2024-11-10 04:36:27,203 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 04:36:27,221 main DEBUG Took 0.009465 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 04:36:27,221 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 04:36:27,222 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 04:36:27,223 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 04:36:27,224 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,230 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 04:36:27,241 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,243 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,243 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,244 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,244 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,245 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,245 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,246 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,247 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,247 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,247 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,248 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,248 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,249 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,249 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,249 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,250 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,250 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,250 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,251 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,251 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:36:27,251 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,252 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 04:36:27,253 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:36:27,254 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 04:36:27,256 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 04:36:27,256 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 04:36:27,257 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 04:36:27,258 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 04:36:27,266 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 04:36:27,268 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 04:36:27,269 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 04:36:27,270 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 04:36:27,270 main DEBUG createAppenders(={Console}) 2024-11-10 04:36:27,271 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-10 04:36:27,271 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 04:36:27,271 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-10 04:36:27,272 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 04:36:27,272 main DEBUG OutputStream closed 2024-11-10 04:36:27,272 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 04:36:27,272 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 04:36:27,273 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-10 04:36:27,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 04:36:27,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 04:36:27,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 04:36:27,342 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 04:36:27,343 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 04:36:27,343 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 04:36:27,343 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 04:36:27,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 04:36:27,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 04:36:27,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 04:36:27,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 04:36:27,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 04:36:27,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 04:36:27,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 04:36:27,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 04:36:27,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 04:36:27,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 04:36:27,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 04:36:27,349 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 04:36:27,350 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-10 04:36:27,350 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 04:36:27,350 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-10T04:36:27,583 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856 2024-11-10 04:36:27,586 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 04:36:27,587 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T04:36:27,595 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-10T04:36:27,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T04:36:27,636 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff, deleteOnExit=true 2024-11-10T04:36:27,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T04:36:27,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/test.cache.data in system properties and HBase conf 2024-11-10T04:36:27,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T04:36:27,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/hadoop.log.dir in system properties and HBase conf 2024-11-10T04:36:27,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T04:36:27,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T04:36:27,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T04:36:27,758 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T04:36:27,877 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:36:27,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:36:27,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T04:36:27,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T04:36:27,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:36:27,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T04:36:27,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T04:36:27,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T04:36:27,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:36:27,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T04:36:27,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/nfs.dump.dir in system properties and HBase conf 2024-11-10T04:36:27,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/java.io.tmpdir in system properties and HBase conf 2024-11-10T04:36:27,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T04:36:27,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T04:36:27,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T04:36:28,679 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T04:36:28,753 INFO [Time-limited test {}] log.Log(170): Logging initialized @2202ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T04:36:28,827 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:36:28,888 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:36:28,910 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:36:28,911 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:36:28,912 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T04:36:28,926 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:36:28,929 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:36:28,930 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:36:29,144 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/java.io.tmpdir/jetty-localhost-43143-hadoop-hdfs-3_4_1-tests_jar-_-any-5728648294078554434/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:36:29,157 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43143} 2024-11-10T04:36:29,158 INFO [Time-limited test {}] server.Server(415): Started @2608ms 2024-11-10T04:36:29,558 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T04:36:29,564 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T04:36:29,565 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T04:36:29,565 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T04:36:29,566 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T04:36:29,566 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/hadoop.log.dir/,AVAILABLE} 2024-11-10T04:36:29,567 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T04:36:29,686 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/java.io.tmpdir/jetty-localhost-39619-hadoop-hdfs-3_4_1-tests_jar-_-any-4838353692161965158/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:36:29,687 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39619} 2024-11-10T04:36:29,687 INFO [Time-limited test {}] server.Server(415): Started @3138ms 2024-11-10T04:36:29,743 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T04:36:30,179 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff/data/data2/current/BP-1091589648-172.17.0.2-1731213388437/current, will proceed with Du for space computation calculation, 2024-11-10T04:36:30,179 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff/data/data1/current/BP-1091589648-172.17.0.2-1731213388437/current, will proceed with Du for space computation calculation, 2024-11-10T04:36:30,222 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T04:36:30,284 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7b77c0b87837496b with lease ID 0xf73e29f65f1c9eb1: Processing first storage report for DS-0bcdabdf-74cc-4c61-ae43-8e5f84ce175c from datanode DatanodeRegistration(127.0.0.1:45227, datanodeUuid=c7686c1b-b994-4981-a73c-a823daec5640, infoPort=37739, infoSecurePort=0, ipcPort=34989, storageInfo=lv=-57;cid=testClusterID;nsid=1974323547;c=1731213388437) 2024-11-10T04:36:30,286 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7b77c0b87837496b with lease ID 0xf73e29f65f1c9eb1: from storage DS-0bcdabdf-74cc-4c61-ae43-8e5f84ce175c node DatanodeRegistration(127.0.0.1:45227, datanodeUuid=c7686c1b-b994-4981-a73c-a823daec5640, infoPort=37739, infoSecurePort=0, ipcPort=34989, storageInfo=lv=-57;cid=testClusterID;nsid=1974323547;c=1731213388437), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-10T04:36:30,286 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7b77c0b87837496b with lease ID 0xf73e29f65f1c9eb1: Processing first storage report for DS-47cd728c-63c3-4bf6-95d9-c2663d22012b from datanode DatanodeRegistration(127.0.0.1:45227, datanodeUuid=c7686c1b-b994-4981-a73c-a823daec5640, infoPort=37739, infoSecurePort=0, ipcPort=34989, storageInfo=lv=-57;cid=testClusterID;nsid=1974323547;c=1731213388437) 2024-11-10T04:36:30,287 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7b77c0b87837496b with lease ID 0xf73e29f65f1c9eb1: from storage DS-47cd728c-63c3-4bf6-95d9-c2663d22012b node DatanodeRegistration(127.0.0.1:45227, datanodeUuid=c7686c1b-b994-4981-a73c-a823daec5640, infoPort=37739, infoSecurePort=0, ipcPort=34989, storageInfo=lv=-57;cid=testClusterID;nsid=1974323547;c=1731213388437), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T04:36:30,310 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856 2024-11-10T04:36:30,392 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff/zookeeper_0, clientPort=57257, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T04:36:30,402 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57257 2024-11-10T04:36:30,413 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:36:30,416 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:36:30,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741825_1001 (size=7) 2024-11-10T04:36:31,046 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8 with version=8 2024-11-10T04:36:31,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/hbase-staging 2024-11-10T04:36:31,139 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T04:36:31,381 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:36:31,392 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:36:31,392 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:36:31,397 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:36:31,397 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:36:31,397 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:36:31,528 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T04:36:31,587 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T04:36:31,595 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T04:36:31,599 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:36:31,625 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 9201 (auto-detected) 2024-11-10T04:36:31,626 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T04:36:31,645 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42929 2024-11-10T04:36:31,666 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42929 connecting to ZooKeeper ensemble=127.0.0.1:57257 2024-11-10T04:36:31,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429290x0, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:36:31,702 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42929-0x1019077678c0000 connected 2024-11-10T04:36:31,729 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:36:31,732 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:36:31,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:36:31,744 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8, hbase.cluster.distributed=false 2024-11-10T04:36:31,765 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:36:31,769 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42929 2024-11-10T04:36:31,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42929 2024-11-10T04:36:31,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42929 2024-11-10T04:36:31,774 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42929 2024-11-10T04:36:31,774 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42929 2024-11-10T04:36:31,880 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d1d9e3766cb1:0 server-side Connection retries=45 2024-11-10T04:36:31,882 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:36:31,882 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:36:31,882 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:36:31,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:36:31,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:36:31,886 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:36:31,888 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:36:31,889 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45693 2024-11-10T04:36:31,891 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45693 connecting to ZooKeeper ensemble=127.0.0.1:57257 2024-11-10T04:36:31,893 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:36:31,897 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:36:31,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456930x0, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:36:31,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456930x0, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:36:31,908 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45693-0x1019077678c0001 connected 2024-11-10T04:36:31,911 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:36:31,920 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:36:31,923 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T04:36:31,929 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:36:31,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45693 2024-11-10T04:36:31,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45693 2024-11-10T04:36:31,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45693 2024-11-10T04:36:31,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45693 2024-11-10T04:36:31,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45693 2024-11-10T04:36:31,949 DEBUG [M:0;d1d9e3766cb1:42929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1d9e3766cb1:42929 2024-11-10T04:36:31,950 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:31,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:36:31,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:36:31,960 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:31,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T04:36:31,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:31,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:31,982 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:36:31,983 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1d9e3766cb1,42929,1731213391190 from backup master directory 2024-11-10T04:36:31,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:31,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:36:31,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:36:31,987 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:36:31,987 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:31,989 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T04:36:31,990 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T04:36:32,045 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/hbase.id] with ID: cb742aa3-047a-478e-8da4-e4764cdd4b8a 2024-11-10T04:36:32,045 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/.tmp/hbase.id 2024-11-10T04:36:32,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741826_1002 (size=42) 2024-11-10T04:36:32,457 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/.tmp/hbase.id]:[hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/hbase.id] 2024-11-10T04:36:32,500 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T04:36:32,505 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T04:36:32,523 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-10T04:36:32,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:32,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:32,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741827_1003 (size=196) 2024-11-10T04:36:32,958 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:36:32,961 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T04:36:32,978 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:32,982 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T04:36:33,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741828_1004 (size=1189) 2024-11-10T04:36:33,029 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store 2024-11-10T04:36:33,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741829_1005 (size=34) 2024-11-10T04:36:33,050 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T04:36:33,053 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:36:33,054 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:36:33,054 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:36:33,054 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:36:33,055 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:36:33,056 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:36:33,056 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:36:33,057 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731213393054Disabling compacts and flushes for region at 1731213393054Disabling writes for close at 1731213393055 (+1 ms)Writing region close event to WAL at 1731213393056 (+1 ms)Closed at 1731213393056 2024-11-10T04:36:33,059 WARN [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/.initializing 2024-11-10T04:36:33,059 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/WALs/d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:33,067 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T04:36:33,081 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C42929%2C1731213391190, suffix=, logDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/WALs/d1d9e3766cb1,42929,1731213391190, archiveDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/oldWALs, maxLogs=10 2024-11-10T04:36:33,103 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/WALs/d1d9e3766cb1,42929,1731213391190/d1d9e3766cb1%2C42929%2C1731213391190.1731213393085, exclude list is [], retry=0 2024-11-10T04:36:33,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45227,DS-0bcdabdf-74cc-4c61-ae43-8e5f84ce175c,DISK] 2024-11-10T04:36:33,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T04:36:33,155 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/WALs/d1d9e3766cb1,42929,1731213391190/d1d9e3766cb1%2C42929%2C1731213391190.1731213393085 2024-11-10T04:36:33,156 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37739:37739)] 2024-11-10T04:36:33,156 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:36:33,157 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:36:33,160 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,161 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,197 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T04:36:33,226 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:33,228 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:36:33,229 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T04:36:33,232 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:33,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:36:33,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T04:36:33,236 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:33,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:36:33,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T04:36:33,239 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:33,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:36:33,241 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,244 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,245 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,250 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,251 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,254 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:36:33,257 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:36:33,261 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:36:33,263 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60671361, jitterRate=-0.09592626988887787}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:36:33,267 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731213393172Initializing all the Stores at 1731213393174 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731213393174Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213393175 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213393175Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213393175Cleaning up temporary data from old regions at 1731213393251 (+76 ms)Region opened successfully at 1731213393267 (+16 ms) 2024-11-10T04:36:33,268 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T04:36:33,300 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1bb4de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:36:33,330 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T04:36:33,341 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T04:36:33,342 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T04:36:33,345 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T04:36:33,346 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T04:36:33,351 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-10T04:36:33,351 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T04:36:33,382 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T04:36:33,393 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T04:36:33,396 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T04:36:33,399 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T04:36:33,400 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T04:36:33,402 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T04:36:33,405 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T04:36:33,408 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T04:36:33,410 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T04:36:33,411 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T04:36:33,413 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T04:36:33,431 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T04:36:33,433 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T04:36:33,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:36:33,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:36:33,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:33,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:33,440 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d1d9e3766cb1,42929,1731213391190, sessionid=0x1019077678c0000, setting cluster-up flag (Was=false) 2024-11-10T04:36:33,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:33,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:33,458 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T04:36:33,460 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:33,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:33,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:33,472 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T04:36:33,474 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:33,480 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T04:36:33,539 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(746): ClusterId : cb742aa3-047a-478e-8da4-e4764cdd4b8a 2024-11-10T04:36:33,542 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:36:33,547 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:36:33,547 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:36:33,550 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:36:33,551 DEBUG [RS:0;d1d9e3766cb1:45693 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f0678e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1d9e3766cb1/172.17.0.2:0 2024-11-10T04:36:33,556 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T04:36:33,566 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1d9e3766cb1:45693 2024-11-10T04:36:33,566 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T04:36:33,569 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:36:33,569 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:36:33,569 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:36:33,572 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(2659): reportForDuty to master=d1d9e3766cb1,42929,1731213391190 with port=45693, startcode=1731213391842 2024-11-10T04:36:33,573 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T04:36:33,579 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1d9e3766cb1,42929,1731213391190 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T04:36:33,583 DEBUG [RS:0;d1d9e3766cb1:45693 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:36:33,586 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:36:33,586 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:36:33,586 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:36:33,587 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:36:33,587 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1d9e3766cb1:0, corePoolSize=10, maxPoolSize=10 2024-11-10T04:36:33,587 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,587 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:36:33,587 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,589 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731213423589 2024-11-10T04:36:33,591 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T04:36:33,592 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T04:36:33,593 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:36:33,594 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T04:36:33,596 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T04:36:33,597 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T04:36:33,597 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T04:36:33,598 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T04:36:33,598 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,601 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:33,602 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T04:36:33,603 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T04:36:33,605 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T04:36:33,606 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T04:36:33,608 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T04:36:33,608 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T04:36:33,610 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731213393609,5,FailOnTimeoutGroup] 2024-11-10T04:36:33,611 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731213393610,5,FailOnTimeoutGroup] 2024-11-10T04:36:33,611 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,611 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T04:36:33,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741831_1007 (size=1321) 2024-11-10T04:36:33,613 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,613 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,618 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39373, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:36:33,623 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42929 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:33,625 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42929 {}] master.ServerManager(517): Registering regionserver=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:33,638 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8 2024-11-10T04:36:33,639 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38385 2024-11-10T04:36:33,639 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:36:33,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:36:33,644 DEBUG [RS:0;d1d9e3766cb1:45693 {}] zookeeper.ZKUtil(111): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:33,644 WARN [RS:0;d1d9e3766cb1:45693 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:36:33,644 INFO [RS:0;d1d9e3766cb1:45693 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T04:36:33,644 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/WALs/d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:33,646 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1d9e3766cb1,45693,1731213391842] 2024-11-10T04:36:33,670 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:36:33,685 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:36:33,690 INFO [RS:0;d1d9e3766cb1:45693 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:36:33,690 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,691 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:36:33,697 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:36:33,699 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,699 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,699 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,699 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,700 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,700 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,700 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:36:33,700 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,700 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,701 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,701 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,701 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,701 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1d9e3766cb1:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:36:33,702 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:36:33,702 DEBUG [RS:0;d1d9e3766cb1:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:36:33,703 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,703 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,703 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,703 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,704 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,704 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,45693,1731213391842-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:36:33,723 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:36:33,725 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,45693,1731213391842-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,725 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,725 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.Replication(171): d1d9e3766cb1,45693,1731213391842 started 2024-11-10T04:36:33,744 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:33,744 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1482): Serving as d1d9e3766cb1,45693,1731213391842, RpcServer on d1d9e3766cb1/172.17.0.2:45693, sessionid=0x1019077678c0001 2024-11-10T04:36:33,745 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:36:33,745 DEBUG [RS:0;d1d9e3766cb1:45693 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:33,746 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,45693,1731213391842' 2024-11-10T04:36:33,746 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:36:33,747 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:36:33,748 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:36:33,748 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:36:33,749 DEBUG [RS:0;d1d9e3766cb1:45693 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:33,749 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1d9e3766cb1,45693,1731213391842' 2024-11-10T04:36:33,749 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:36:33,749 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:36:33,750 DEBUG [RS:0;d1d9e3766cb1:45693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:36:33,750 INFO [RS:0;d1d9e3766cb1:45693 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:36:33,750 INFO [RS:0;d1d9e3766cb1:45693 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:36:33,856 INFO [RS:0;d1d9e3766cb1:45693 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T04:36:33,859 INFO [RS:0;d1d9e3766cb1:45693 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C45693%2C1731213391842, suffix=, logDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/WALs/d1d9e3766cb1,45693,1731213391842, archiveDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/oldWALs, maxLogs=32 2024-11-10T04:36:33,875 DEBUG [RS:0;d1d9e3766cb1:45693 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/WALs/d1d9e3766cb1,45693,1731213391842/d1d9e3766cb1%2C45693%2C1731213391842.1731213393861, exclude list is [], retry=0 2024-11-10T04:36:33,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45227,DS-0bcdabdf-74cc-4c61-ae43-8e5f84ce175c,DISK] 2024-11-10T04:36:33,883 INFO [RS:0;d1d9e3766cb1:45693 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/WALs/d1d9e3766cb1,45693,1731213391842/d1d9e3766cb1%2C45693%2C1731213391842.1731213393861 2024-11-10T04:36:33,884 DEBUG [RS:0;d1d9e3766cb1:45693 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37739:37739)] 2024-11-10T04:36:34,014 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T04:36:34,015 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8 2024-11-10T04:36:34,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741833_1009 (size=32) 2024-11-10T04:36:34,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:36:34,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:36:34,431 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:36:34,431 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:34,432 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:36:34,432 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:36:34,435 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:36:34,435 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:34,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:36:34,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:36:34,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:36:34,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:34,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:36:34,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:36:34,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:36:34,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:34,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:36:34,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:36:34,444 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740 2024-11-10T04:36:34,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740 2024-11-10T04:36:34,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:36:34,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:36:34,449 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:36:34,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:36:34,454 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:36:34,455 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59623231, jitterRate=-0.11154462397098541}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:36:34,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731213394426Initializing all the Stores at 1731213394428 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731213394428Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731213394428Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213394428Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731213394428Cleaning up temporary data from old regions at 1731213394448 (+20 ms)Region opened successfully at 1731213394457 (+9 ms) 2024-11-10T04:36:34,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:36:34,458 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:36:34,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:36:34,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:36:34,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:36:34,459 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:36:34,459 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731213394458Disabling compacts and flushes for region at 1731213394458Disabling writes for close at 1731213394458Writing region close event to WAL at 1731213394459 (+1 ms)Closed at 1731213394459 2024-11-10T04:36:34,462 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:36:34,462 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T04:36:34,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:36:34,477 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:36:34,480 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T04:36:34,632 DEBUG [d1d9e3766cb1:42929 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T04:36:34,645 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:34,650 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,45693,1731213391842, state=OPENING 2024-11-10T04:36:34,655 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T04:36:34,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:34,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:36:34,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:36:34,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:36:34,659 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:36:34,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:36:34,836 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:36:34,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42981, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:36:34,850 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T04:36:34,851 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T04:36:34,851 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T04:36:34,855 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1d9e3766cb1%2C45693%2C1731213391842.meta, suffix=.meta, logDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/WALs/d1d9e3766cb1,45693,1731213391842, archiveDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/oldWALs, maxLogs=32 2024-11-10T04:36:34,873 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/WALs/d1d9e3766cb1,45693,1731213391842/d1d9e3766cb1%2C45693%2C1731213391842.meta.1731213394857.meta, exclude list is [], retry=0 2024-11-10T04:36:34,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45227,DS-0bcdabdf-74cc-4c61-ae43-8e5f84ce175c,DISK] 2024-11-10T04:36:34,879 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/WALs/d1d9e3766cb1,45693,1731213391842/d1d9e3766cb1%2C45693%2C1731213391842.meta.1731213394857.meta 2024-11-10T04:36:34,880 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37739:37739)] 2024-11-10T04:36:34,880 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:36:34,882 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T04:36:34,884 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T04:36:34,890 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T04:36:34,896 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T04:36:34,896 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:36:34,897 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T04:36:34,897 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T04:36:34,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T04:36:34,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:36:34,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:34,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:36:34,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T04:36:34,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:36:34,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:34,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:36:34,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T04:36:34,906 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:36:34,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:34,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:36:34,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T04:36:34,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:36:34,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:34,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:36:34,909 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:36:34,911 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740 2024-11-10T04:36:34,914 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740 2024-11-10T04:36:34,917 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:36:34,917 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:36:34,918 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:36:34,921 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:36:34,923 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61275767, jitterRate=-0.08691991865634918}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:36:34,923 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T04:36:34,925 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731213394897Writing region info on filesystem at 1731213394898 (+1 ms)Initializing all the Stores at 1731213394899 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731213394900 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731213394900Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213394900Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731213394900Cleaning up temporary data from old regions at 1731213394917 (+17 ms)Running coprocessor post-open hooks at 1731213394923 (+6 ms)Region opened successfully at 1731213394925 (+2 ms) 2024-11-10T04:36:34,933 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731213394827 2024-11-10T04:36:34,947 DEBUG [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T04:36:34,947 INFO [RS_OPEN_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T04:36:34,949 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:34,951 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1d9e3766cb1,45693,1731213391842, state=OPEN 2024-11-10T04:36:34,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:36:34,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:36:34,966 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:36:34,966 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:36:34,966 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:34,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T04:36:34,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d1d9e3766cb1,45693,1731213391842 in 305 msec 2024-11-10T04:36:34,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T04:36:34,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 503 msec 2024-11-10T04:36:34,978 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:36:34,978 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T04:36:34,998 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:35,000 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:35,021 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:35,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59981, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:35,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5380 sec 2024-11-10T04:36:35,048 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731213395048, completionTime=-1 2024-11-10T04:36:35,052 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T04:36:35,053 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T04:36:35,086 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T04:36:35,086 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731213455086 2024-11-10T04:36:35,086 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731213515086 2024-11-10T04:36:35,086 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-11-10T04:36:35,090 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42929,1731213391190-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:35,090 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42929,1731213391190-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:35,090 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42929,1731213391190-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:35,092 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1d9e3766cb1:42929, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:35,092 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:35,093 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:35,099 DEBUG [master/d1d9e3766cb1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T04:36:35,118 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.131sec 2024-11-10T04:36:35,119 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T04:36:35,121 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T04:36:35,122 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T04:36:35,122 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T04:36:35,122 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T04:36:35,123 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42929,1731213391190-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:36:35,124 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42929,1731213391190-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T04:36:35,132 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T04:36:35,132 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T04:36:35,133 INFO [master/d1d9e3766cb1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1d9e3766cb1,42929,1731213391190-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:36:35,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:35,150 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T04:36:35,150 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T04:36:35,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:35,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:35,165 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc39be2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:35,175 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:35,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:35,178 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:35,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:35,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:35,220 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=18, ProcessCount=11, AvailableMemoryMB=7141 2024-11-10T04:36:35,229 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:36:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:35,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78a466cf 2024-11-10T04:36:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:36:35,240 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:36:35,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T04:36:35,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:36:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T04:36:35,277 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:36:35,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T04:36:35,281 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:35,283 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:36:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:36:35,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741835_1011 (size=959) 2024-11-10T04:36:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:36:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:36:35,703 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8 2024-11-10T04:36:35,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741836_1012 (size=53) 2024-11-10T04:36:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:36:36,112 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:36:36,113 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6d2efd97fd4ccb14e877b384767c0ffc, disabling compactions & flushes 2024-11-10T04:36:36,113 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:36,113 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:36,113 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. after waiting 0 ms 2024-11-10T04:36:36,113 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:36,113 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:36,113 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6d2efd97fd4ccb14e877b384767c0ffc: Waiting for close lock at 1731213396112Disabling compacts and flushes for region at 1731213396112Disabling writes for close at 1731213396113 (+1 ms)Writing region close event to WAL at 1731213396113Closed at 1731213396113 2024-11-10T04:36:36,115 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:36:36,121 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731213396115"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731213396115"}]},"ts":"1731213396115"} 2024-11-10T04:36:36,124 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:36:36,126 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:36:36,129 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213396126"}]},"ts":"1731213396126"} 2024-11-10T04:36:36,133 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T04:36:36,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d2efd97fd4ccb14e877b384767c0ffc, ASSIGN}] 2024-11-10T04:36:36,138 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d2efd97fd4ccb14e877b384767c0ffc, ASSIGN 2024-11-10T04:36:36,139 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d2efd97fd4ccb14e877b384767c0ffc, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,45693,1731213391842; forceNewPlan=false, retain=false 2024-11-10T04:36:36,291 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6d2efd97fd4ccb14e877b384767c0ffc, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:36,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d2efd97fd4ccb14e877b384767c0ffc, ASSIGN because future has completed 2024-11-10T04:36:36,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:36:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:36:36,456 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:36,456 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:36:36,457 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,457 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:36:36,457 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,457 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,459 INFO [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,462 INFO [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d2efd97fd4ccb14e877b384767c0ffc columnFamilyName A 2024-11-10T04:36:36,462 DEBUG [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:36,463 INFO [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] regionserver.HStore(327): Store=6d2efd97fd4ccb14e877b384767c0ffc/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:36:36,463 INFO [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,465 INFO [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d2efd97fd4ccb14e877b384767c0ffc columnFamilyName B 2024-11-10T04:36:36,465 DEBUG [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:36,466 INFO [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] regionserver.HStore(327): Store=6d2efd97fd4ccb14e877b384767c0ffc/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:36:36,466 INFO [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,468 INFO [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d2efd97fd4ccb14e877b384767c0ffc columnFamilyName C 2024-11-10T04:36:36,468 DEBUG [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:36:36,468 INFO [StoreOpener-6d2efd97fd4ccb14e877b384767c0ffc-1 {}] regionserver.HStore(327): Store=6d2efd97fd4ccb14e877b384767c0ffc/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:36:36,469 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,470 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,471 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,472 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,472 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,473 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:36:36,476 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,479 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:36:36,480 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6d2efd97fd4ccb14e877b384767c0ffc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60700361, jitterRate=-0.09549413621425629}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:36:36,480 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:36,481 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6d2efd97fd4ccb14e877b384767c0ffc: Running coprocessor pre-open hook at 1731213396457Writing region info on filesystem at 1731213396457Initializing all the Stores at 1731213396459 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213396459Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213396459Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213396459Cleaning up temporary data from old regions at 1731213396473 (+14 ms)Running coprocessor post-open hooks at 1731213396480 (+7 ms)Region opened successfully at 1731213396481 (+1 ms) 2024-11-10T04:36:36,483 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., pid=6, masterSystemTime=1731213396450 2024-11-10T04:36:36,486 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:36,486 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:36,487 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6d2efd97fd4ccb14e877b384767c0ffc, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:36,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:36:36,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T04:36:36,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 in 197 msec 2024-11-10T04:36:36,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T04:36:36,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d2efd97fd4ccb14e877b384767c0ffc, ASSIGN in 362 msec 2024-11-10T04:36:36,502 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:36:36,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213396502"}]},"ts":"1731213396502"} 2024-11-10T04:36:36,506 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T04:36:36,507 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:36:36,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2380 sec 2024-11-10T04:36:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:36:37,432 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:37,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,440 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e00daac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,443 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f43f46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,445 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,451 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:36:37,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,42929,1731213391190 2024-11-10T04:36:37,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1907f2c6 2024-11-10T04:36:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:36:37,456 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:36:37,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,462 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2311693d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,464 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ce3c82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,467 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2886558d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,470 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e6a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,472 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,472 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,473 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36644156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,477 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,480 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f5a6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,483 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a37cd4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,485 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6740e60c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,488 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@418d3001, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,491 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af3e7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b35163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,497 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94111c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,499 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13185de7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,502 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b456016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,504 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc2de1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:36:37,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:36:37,506 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:36:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:36:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:36:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69080074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:36:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:36:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:37,509 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:36:37,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67e6bf95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:36:37,515 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,515 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,516 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,516 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,516 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,516 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,517 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,517 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:36:37,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:36:37,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:36:37,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:36:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:36:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:36:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:36:37,536 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:36:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:36:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:36:37,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:36:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:36:37,550 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:36:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T04:36:37,562 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T04:36:37,565 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:37,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:37,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:37,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:36:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T04:36:37,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/af089e3731ee4e17858e433fda8f27b6 is 50, key is test_row_0/A:col10/1731213397599/Put/seqid=0 2024-11-10T04:36:37,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741837_1013 (size=12001) 2024-11-10T04:36:37,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:37,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:37,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:37,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:37,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:37,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213457791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213457797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213457798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213457800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213457803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T04:36:37,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:37,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:37,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:37,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:37,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213457942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213457944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213457944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:37,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213457944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213457943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:38,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/af089e3731ee4e17858e433fda8f27b6 2024-11-10T04:36:38,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213458154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213458154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213458154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213458162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213458162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T04:36:38,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/476370440ad445d1bc77d6585eeea561 is 50, key is test_row_0/B:col10/1731213397599/Put/seqid=0 2024-11-10T04:36:38,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741838_1014 (size=12001) 2024-11-10T04:36:38,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/476370440ad445d1bc77d6585eeea561 2024-11-10T04:36:38,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:38,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:38,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/420ec416a3be4f35887696da839de68b is 50, key is test_row_0/C:col10/1731213397599/Put/seqid=0 2024-11-10T04:36:38,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741839_1015 (size=12001) 2024-11-10T04:36:38,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/420ec416a3be4f35887696da839de68b 2024-11-10T04:36:38,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/af089e3731ee4e17858e433fda8f27b6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/af089e3731ee4e17858e433fda8f27b6 2024-11-10T04:36:38,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/af089e3731ee4e17858e433fda8f27b6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T04:36:38,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/476370440ad445d1bc77d6585eeea561 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/476370440ad445d1bc77d6585eeea561 2024-11-10T04:36:38,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/476370440ad445d1bc77d6585eeea561, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T04:36:38,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/420ec416a3be4f35887696da839de68b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/420ec416a3be4f35887696da839de68b 2024-11-10T04:36:38,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:38,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:38,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/420ec416a3be4f35887696da839de68b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T04:36:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 6d2efd97fd4ccb14e877b384767c0ffc in 809ms, sequenceid=13, compaction requested=false 2024-11-10T04:36:38,425 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T04:36:38,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:38,478 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:38,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:36:38,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213458497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213458504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213458504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213458510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/81c1595d72084ed8b97599c267fd8443 is 50, key is test_row_0/A:col10/1731213397790/Put/seqid=0 2024-11-10T04:36:38,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213458511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741840_1016 (size=14341) 2024-11-10T04:36:38,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/81c1595d72084ed8b97599c267fd8443 2024-11-10T04:36:38,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/254e14a3c704487e862cda7bcc8b9893 is 50, key is test_row_0/B:col10/1731213397790/Put/seqid=0 2024-11-10T04:36:38,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741841_1017 (size=12001) 2024-11-10T04:36:38,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213458622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213458623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213458624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213458624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213458632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T04:36:38,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:38,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:38,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213458842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213458844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213458844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213458844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213458845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:38,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:38,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:38,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:38,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/254e14a3c704487e862cda7bcc8b9893 2024-11-10T04:36:39,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/dd31c72a7676467795a75d5c8eaf68af is 50, key is test_row_0/C:col10/1731213397790/Put/seqid=0 2024-11-10T04:36:39,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:39,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741842_1018 (size=12001) 2024-11-10T04:36:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213459152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213459152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213459152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213459162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213459162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:39,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:39,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:39,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:39,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:39,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:39,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:39,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:39,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/dd31c72a7676467795a75d5c8eaf68af 2024-11-10T04:36:39,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/81c1595d72084ed8b97599c267fd8443 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/81c1595d72084ed8b97599c267fd8443 2024-11-10T04:36:39,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/81c1595d72084ed8b97599c267fd8443, entries=200, sequenceid=39, filesize=14.0 K 2024-11-10T04:36:39,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/254e14a3c704487e862cda7bcc8b9893 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/254e14a3c704487e862cda7bcc8b9893 2024-11-10T04:36:39,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/254e14a3c704487e862cda7bcc8b9893, entries=150, sequenceid=39, filesize=11.7 K 2024-11-10T04:36:39,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:39,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:39,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/dd31c72a7676467795a75d5c8eaf68af as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dd31c72a7676467795a75d5c8eaf68af 2024-11-10T04:36:39,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dd31c72a7676467795a75d5c8eaf68af, entries=150, sequenceid=39, filesize=11.7 K 2024-11-10T04:36:39,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6d2efd97fd4ccb14e877b384767c0ffc in 1072ms, sequenceid=39, compaction requested=false 2024-11-10T04:36:39,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:39,664 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T04:36:39,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T04:36:39,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:39,670 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T04:36:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:39,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:39,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/d2639fee20464c5a858c822eb038d484 is 50, key is test_row_0/A:col10/1731213398505/Put/seqid=0 2024-11-10T04:36:39,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741843_1019 (size=12001) 2024-11-10T04:36:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T04:36:39,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213459767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213459770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213459772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213459775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213459780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213459882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213459892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213459893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213459893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:39,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213459902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213460092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213460101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,102 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/d2639fee20464c5a858c822eb038d484 2024-11-10T04:36:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213460102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213460111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213460112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/759866fcd2a1429fab0068fec47ed337 is 50, key is test_row_0/B:col10/1731213398505/Put/seqid=0 2024-11-10T04:36:40,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741844_1020 (size=12001) 2024-11-10T04:36:40,144 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/759866fcd2a1429fab0068fec47ed337 2024-11-10T04:36:40,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/6f43e1a162af408987dc7f2de2afb1ec is 50, key is test_row_0/C:col10/1731213398505/Put/seqid=0 2024-11-10T04:36:40,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741845_1021 (size=12001) 2024-11-10T04:36:40,197 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/6f43e1a162af408987dc7f2de2afb1ec 2024-11-10T04:36:40,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/d2639fee20464c5a858c822eb038d484 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/d2639fee20464c5a858c822eb038d484 2024-11-10T04:36:40,232 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/d2639fee20464c5a858c822eb038d484, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T04:36:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/759866fcd2a1429fab0068fec47ed337 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/759866fcd2a1429fab0068fec47ed337 2024-11-10T04:36:40,251 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/759866fcd2a1429fab0068fec47ed337, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T04:36:40,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/6f43e1a162af408987dc7f2de2afb1ec as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/6f43e1a162af408987dc7f2de2afb1ec 2024-11-10T04:36:40,265 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/6f43e1a162af408987dc7f2de2afb1ec, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T04:36:40,268 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6d2efd97fd4ccb14e877b384767c0ffc in 599ms, sequenceid=50, compaction requested=true 2024-11-10T04:36:40,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:40,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:40,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T04:36:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T04:36:40,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T04:36:40,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7040 sec 2024-11-10T04:36:40,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.7260 sec 2024-11-10T04:36:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:40,407 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:40,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T04:36:40,409 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T04:36:40,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8e0dbb5971394ba48eba19d1c271f40e is 50, key is test_row_0/A:col10/1731213399772/Put/seqid=0 2024-11-10T04:36:40,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213460422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213460422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213460426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213460427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213460429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741846_1022 (size=12001) 2024-11-10T04:36:40,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8e0dbb5971394ba48eba19d1c271f40e 2024-11-10T04:36:40,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/63273bd9878f44c182ce6a3fbbd6700a is 50, key is test_row_0/B:col10/1731213399772/Put/seqid=0 2024-11-10T04:36:40,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741847_1023 (size=12001) 2024-11-10T04:36:40,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/63273bd9878f44c182ce6a3fbbd6700a 2024-11-10T04:36:40,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213460542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213460542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213460543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/faac07d70f76455992ba93ef0a1e2373 is 50, key is test_row_0/C:col10/1731213399772/Put/seqid=0 2024-11-10T04:36:40,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741848_1024 (size=12001) 2024-11-10T04:36:40,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/faac07d70f76455992ba93ef0a1e2373 2024-11-10T04:36:40,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8e0dbb5971394ba48eba19d1c271f40e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8e0dbb5971394ba48eba19d1c271f40e 2024-11-10T04:36:40,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8e0dbb5971394ba48eba19d1c271f40e, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T04:36:40,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/63273bd9878f44c182ce6a3fbbd6700a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/63273bd9878f44c182ce6a3fbbd6700a 2024-11-10T04:36:40,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/63273bd9878f44c182ce6a3fbbd6700a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T04:36:40,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/faac07d70f76455992ba93ef0a1e2373 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/faac07d70f76455992ba93ef0a1e2373 2024-11-10T04:36:40,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/faac07d70f76455992ba93ef0a1e2373, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T04:36:40,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 6d2efd97fd4ccb14e877b384767c0ffc in 229ms, sequenceid=77, compaction requested=true 2024-11-10T04:36:40,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:40,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:36:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:36:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:36:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:40,645 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:36:40,645 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:36:40,650 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:36:40,650 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:36:40,652 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/A is initiating minor compaction (all files) 2024-11-10T04:36:40,652 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/B is initiating minor compaction (all files) 2024-11-10T04:36:40,653 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/A in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:40,653 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/B in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:40,653 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/af089e3731ee4e17858e433fda8f27b6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/81c1595d72084ed8b97599c267fd8443, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/d2639fee20464c5a858c822eb038d484, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8e0dbb5971394ba48eba19d1c271f40e] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=49.2 K 2024-11-10T04:36:40,653 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/476370440ad445d1bc77d6585eeea561, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/254e14a3c704487e862cda7bcc8b9893, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/759866fcd2a1429fab0068fec47ed337, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/63273bd9878f44c182ce6a3fbbd6700a] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=46.9 K 2024-11-10T04:36:40,655 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting af089e3731ee4e17858e433fda8f27b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731213397542 2024-11-10T04:36:40,655 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 476370440ad445d1bc77d6585eeea561, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731213397542 2024-11-10T04:36:40,656 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 254e14a3c704487e862cda7bcc8b9893, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731213397790 2024-11-10T04:36:40,656 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c1595d72084ed8b97599c267fd8443, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731213397790 2024-11-10T04:36:40,657 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 759866fcd2a1429fab0068fec47ed337, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731213398502 2024-11-10T04:36:40,657 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2639fee20464c5a858c822eb038d484, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731213398502 2024-11-10T04:36:40,657 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 63273bd9878f44c182ce6a3fbbd6700a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731213399772 2024-11-10T04:36:40,659 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e0dbb5971394ba48eba19d1c271f40e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731213399772 2024-11-10T04:36:40,706 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#A#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:40,707 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/db471b6d187c43559ee33c8c23452db5 is 50, key is test_row_0/A:col10/1731213399772/Put/seqid=0 2024-11-10T04:36:40,711 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#B#compaction#13 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:40,712 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/89a5c21548234bfeb6cb62357ac4b4a4 is 50, key is test_row_0/B:col10/1731213399772/Put/seqid=0 2024-11-10T04:36:40,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741849_1025 (size=12139) 2024-11-10T04:36:40,746 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/db471b6d187c43559ee33c8c23452db5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/db471b6d187c43559ee33c8c23452db5 2024-11-10T04:36:40,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741850_1026 (size=12139) 2024-11-10T04:36:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:40,758 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:40,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:36:40,763 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/89a5c21548234bfeb6cb62357ac4b4a4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/89a5c21548234bfeb6cb62357ac4b4a4 2024-11-10T04:36:40,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/cb3bb56a23014f238079ba66f22baac1 is 50, key is test_row_0/A:col10/1731213400753/Put/seqid=0 2024-11-10T04:36:40,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741851_1027 (size=16681) 2024-11-10T04:36:40,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/cb3bb56a23014f238079ba66f22baac1 2024-11-10T04:36:40,792 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/B of 6d2efd97fd4ccb14e877b384767c0ffc into 89a5c21548234bfeb6cb62357ac4b4a4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:40,792 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/A of 6d2efd97fd4ccb14e877b384767c0ffc into db471b6d187c43559ee33c8c23452db5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:40,792 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:40,792 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:40,796 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/B, priority=12, startTime=1731213400642; duration=0sec 2024-11-10T04:36:40,796 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/A, priority=12, startTime=1731213400639; duration=0sec 2024-11-10T04:36:40,797 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:40,797 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:A 2024-11-10T04:36:40,797 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:36:40,799 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:40,799 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:B 2024-11-10T04:36:40,804 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:36:40,804 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/C is initiating minor compaction (all files) 2024-11-10T04:36:40,804 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/C in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:40,804 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/420ec416a3be4f35887696da839de68b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dd31c72a7676467795a75d5c8eaf68af, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/6f43e1a162af408987dc7f2de2afb1ec, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/faac07d70f76455992ba93ef0a1e2373] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=46.9 K 2024-11-10T04:36:40,805 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 420ec416a3be4f35887696da839de68b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731213397542 2024-11-10T04:36:40,806 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd31c72a7676467795a75d5c8eaf68af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731213397790 2024-11-10T04:36:40,807 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f43e1a162af408987dc7f2de2afb1ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731213398502 2024-11-10T04:36:40,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/ccdc449167ae460e9e802889bfbe762d is 50, key is test_row_0/B:col10/1731213400753/Put/seqid=0 2024-11-10T04:36:40,808 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting faac07d70f76455992ba93ef0a1e2373, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731213399772 2024-11-10T04:36:40,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741852_1028 (size=12001) 2024-11-10T04:36:40,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/ccdc449167ae460e9e802889bfbe762d 2024-11-10T04:36:40,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/dc6a418717ee4d3585ba6788b4b955f2 is 50, key is test_row_0/C:col10/1731213400753/Put/seqid=0 2024-11-10T04:36:40,864 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#C#compaction#16 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:40,865 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/40f1b730db9d4a3d8d4d7755a452b0f7 is 50, key is test_row_0/C:col10/1731213399772/Put/seqid=0 2024-11-10T04:36:40,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213460874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741853_1029 (size=12001) 2024-11-10T04:36:40,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213460877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213460877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/dc6a418717ee4d3585ba6788b4b955f2 2024-11-10T04:36:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/cb3bb56a23014f238079ba66f22baac1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/cb3bb56a23014f238079ba66f22baac1 2024-11-10T04:36:40,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741854_1030 (size=12139) 2024-11-10T04:36:40,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/cb3bb56a23014f238079ba66f22baac1, entries=250, sequenceid=89, filesize=16.3 K 2024-11-10T04:36:40,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/ccdc449167ae460e9e802889bfbe762d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ccdc449167ae460e9e802889bfbe762d 2024-11-10T04:36:40,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ccdc449167ae460e9e802889bfbe762d, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T04:36:40,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/dc6a418717ee4d3585ba6788b4b955f2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dc6a418717ee4d3585ba6788b4b955f2 2024-11-10T04:36:40,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213460941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213460941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:40,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dc6a418717ee4d3585ba6788b4b955f2, entries=150, sequenceid=89, filesize=11.7 K 2024-11-10T04:36:40,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6d2efd97fd4ccb14e877b384767c0ffc in 197ms, sequenceid=89, compaction requested=false 2024-11-10T04:36:40,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:40,989 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:40,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:36:41,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/a78a019721204882a2eac225fe6b5eb2 is 50, key is test_row_0/A:col10/1731213400982/Put/seqid=0 2024-11-10T04:36:41,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741855_1031 (size=14341) 2024-11-10T04:36:41,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/a78a019721204882a2eac225fe6b5eb2 2024-11-10T04:36:41,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213461017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213461019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213461023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/c7ae9ac5b2be4b7f985b5abb9905c0e0 is 50, key is test_row_0/B:col10/1731213400982/Put/seqid=0 2024-11-10T04:36:41,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741856_1032 (size=12001) 2024-11-10T04:36:41,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/c7ae9ac5b2be4b7f985b5abb9905c0e0 2024-11-10T04:36:41,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/9c546730c2684f7ba157025bc52cfcab is 50, key is test_row_0/C:col10/1731213400982/Put/seqid=0 2024-11-10T04:36:41,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741857_1033 (size=12001) 2024-11-10T04:36:41,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/9c546730c2684f7ba157025bc52cfcab 2024-11-10T04:36:41,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/a78a019721204882a2eac225fe6b5eb2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a78a019721204882a2eac225fe6b5eb2 2024-11-10T04:36:41,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a78a019721204882a2eac225fe6b5eb2, entries=200, sequenceid=116, filesize=14.0 K 2024-11-10T04:36:41,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/c7ae9ac5b2be4b7f985b5abb9905c0e0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c7ae9ac5b2be4b7f985b5abb9905c0e0 2024-11-10T04:36:41,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213461131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213461132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213461132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c7ae9ac5b2be4b7f985b5abb9905c0e0, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T04:36:41,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/9c546730c2684f7ba157025bc52cfcab as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/9c546730c2684f7ba157025bc52cfcab 2024-11-10T04:36:41,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/9c546730c2684f7ba157025bc52cfcab, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T04:36:41,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 6d2efd97fd4ccb14e877b384767c0ffc in 176ms, sequenceid=116, compaction requested=true 2024-11-10T04:36:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:A, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:36:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:41,166 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:B, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:36:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:36:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T04:36:41,168 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:41,168 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/A is initiating minor compaction (all files) 2024-11-10T04:36:41,168 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/A in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:41,168 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/db471b6d187c43559ee33c8c23452db5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/cb3bb56a23014f238079ba66f22baac1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a78a019721204882a2eac225fe6b5eb2] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=42.1 K 2024-11-10T04:36:41,169 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting db471b6d187c43559ee33c8c23452db5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731213399772 2024-11-10T04:36:41,170 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cb3bb56a23014f238079ba66f22baac1, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731213400423 2024-11-10T04:36:41,171 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a78a019721204882a2eac225fe6b5eb2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213400853 2024-11-10T04:36:41,199 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#A#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:41,200 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/6655f07490f7479abde69388c75769d0 is 50, key is test_row_0/A:col10/1731213400982/Put/seqid=0 2024-11-10T04:36:41,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741858_1034 (size=12241) 2024-11-10T04:36:41,321 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/40f1b730db9d4a3d8d4d7755a452b0f7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/40f1b730db9d4a3d8d4d7755a452b0f7 2024-11-10T04:36:41,333 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/C of 6d2efd97fd4ccb14e877b384767c0ffc into 40f1b730db9d4a3d8d4d7755a452b0f7(size=11.9 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:41,333 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:41,334 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/C, priority=12, startTime=1731213400642; duration=0sec 2024-11-10T04:36:41,334 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T04:36:41,334 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:36:41,334 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:36:41,334 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:41,338 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:41,338 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/B is initiating minor compaction (all files) 2024-11-10T04:36:41,338 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/B in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:41,338 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/89a5c21548234bfeb6cb62357ac4b4a4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ccdc449167ae460e9e802889bfbe762d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c7ae9ac5b2be4b7f985b5abb9905c0e0] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=35.3 K 2024-11-10T04:36:41,339 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89a5c21548234bfeb6cb62357ac4b4a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731213399772 2024-11-10T04:36:41,342 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccdc449167ae460e9e802889bfbe762d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731213400753 2024-11-10T04:36:41,343 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7ae9ac5b2be4b7f985b5abb9905c0e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213400853 2024-11-10T04:36:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:41,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:41,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T04:36:41,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/c5ad68ea14c94f4e822a7d3c86b8e5c3 is 50, key is test_row_0/A:col10/1731213401016/Put/seqid=0 2024-11-10T04:36:41,368 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#B#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:41,370 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/0e2742f6ec3d4bdc8638b2ecce918fff is 50, key is test_row_0/B:col10/1731213400982/Put/seqid=0 2024-11-10T04:36:41,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741859_1035 (size=12101) 2024-11-10T04:36:41,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213461429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213461434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741860_1036 (size=12241) 2024-11-10T04:36:41,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213461435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,449 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/0e2742f6ec3d4bdc8638b2ecce918fff as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/0e2742f6ec3d4bdc8638b2ecce918fff 2024-11-10T04:36:41,464 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/B of 6d2efd97fd4ccb14e877b384767c0ffc into 0e2742f6ec3d4bdc8638b2ecce918fff(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:41,464 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:41,464 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/B, priority=13, startTime=1731213401166; duration=0sec 2024-11-10T04:36:41,464 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:41,464 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:B 2024-11-10T04:36:41,465 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:41,467 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:41,467 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/C is initiating minor compaction (all files) 2024-11-10T04:36:41,467 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/C in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:41,467 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/40f1b730db9d4a3d8d4d7755a452b0f7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dc6a418717ee4d3585ba6788b4b955f2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/9c546730c2684f7ba157025bc52cfcab] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=35.3 K 2024-11-10T04:36:41,468 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40f1b730db9d4a3d8d4d7755a452b0f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731213399772 2024-11-10T04:36:41,469 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc6a418717ee4d3585ba6788b4b955f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731213400753 2024-11-10T04:36:41,470 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c546730c2684f7ba157025bc52cfcab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213400853 2024-11-10T04:36:41,488 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:41,489 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/51a89506e538477ab8e80b4c94b78bfb is 50, key is test_row_0/C:col10/1731213400982/Put/seqid=0 2024-11-10T04:36:41,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741861_1037 (size=12241) 2024-11-10T04:36:41,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213461542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213461542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213461551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,584 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T04:36:41,585 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T04:36:41,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:36:41,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T04:36:41,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T04:36:41,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T04:36:41,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T04:36:41,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T04:36:41,633 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/6655f07490f7479abde69388c75769d0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6655f07490f7479abde69388c75769d0 2024-11-10T04:36:41,645 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/A of 6d2efd97fd4ccb14e877b384767c0ffc into 6655f07490f7479abde69388c75769d0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:41,645 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:41,645 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/A, priority=13, startTime=1731213401166; duration=0sec 2024-11-10T04:36:41,645 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:41,645 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:A 2024-11-10T04:36:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T04:36:41,742 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:41,743 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T04:36:41,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T04:36:41,749 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:41,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213461752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213461752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213461762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/c5ad68ea14c94f4e822a7d3c86b8e5c3 2024-11-10T04:36:41,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/b66aaac446bf43edbfd353ea89a2be38 is 50, key is test_row_0/B:col10/1731213401016/Put/seqid=0 2024-11-10T04:36:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T04:36:41,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741862_1038 (size=12101) 2024-11-10T04:36:41,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T04:36:41,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:41,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:41,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:41,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:41,933 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/51a89506e538477ab8e80b4c94b78bfb as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/51a89506e538477ab8e80b4c94b78bfb 2024-11-10T04:36:41,943 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/C of 6d2efd97fd4ccb14e877b384767c0ffc into 51a89506e538477ab8e80b4c94b78bfb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:41,944 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:41,944 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/C, priority=13, startTime=1731213401166; duration=0sec 2024-11-10T04:36:41,944 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:41,944 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:36:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213461962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213461962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:41,968 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 6d2efd97fd4ccb14e877b384767c0ffc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:36:41,968 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 6d2efd97fd4ccb14e877b384767c0ffc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:36:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T04:36:42,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213462061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:42,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213462062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:42,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T04:36:42,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:42,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213462071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:42,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T04:36:42,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:42,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/b66aaac446bf43edbfd353ea89a2be38 2024-11-10T04:36:42,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/ae926e56c92f46b48f3174ab9ae7257b is 50, key is test_row_0/C:col10/1731213401016/Put/seqid=0 2024-11-10T04:36:42,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741863_1039 (size=12101) 2024-11-10T04:36:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T04:36:42,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T04:36:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:42,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T04:36:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213462571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213462582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213462582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:42,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:42,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/ae926e56c92f46b48f3174ab9ae7257b 2024-11-10T04:36:42,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T04:36:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/c5ad68ea14c94f4e822a7d3c86b8e5c3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c5ad68ea14c94f4e822a7d3c86b8e5c3 2024-11-10T04:36:42,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c5ad68ea14c94f4e822a7d3c86b8e5c3, entries=150, sequenceid=131, filesize=11.8 K 2024-11-10T04:36:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/b66aaac446bf43edbfd353ea89a2be38 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b66aaac446bf43edbfd353ea89a2be38 2024-11-10T04:36:42,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b66aaac446bf43edbfd353ea89a2be38, entries=150, sequenceid=131, filesize=11.8 K 2024-11-10T04:36:42,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/ae926e56c92f46b48f3174ab9ae7257b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/ae926e56c92f46b48f3174ab9ae7257b 2024-11-10T04:36:42,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/ae926e56c92f46b48f3174ab9ae7257b, entries=150, sequenceid=131, filesize=11.8 K 2024-11-10T04:36:42,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 6d2efd97fd4ccb14e877b384767c0ffc in 1386ms, sequenceid=131, compaction requested=false 2024-11-10T04:36:42,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:42,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T04:36:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:42,840 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T04:36:42,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/5efb4edc2675426ea21add4c521a9265 is 50, key is test_row_0/A:col10/1731213401431/Put/seqid=0 2024-11-10T04:36:42,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741864_1040 (size=12151) 2024-11-10T04:36:42,859 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/5efb4edc2675426ea21add4c521a9265 2024-11-10T04:36:42,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/5f07af19421e4976af81c7ee83a825d2 is 50, key is test_row_0/B:col10/1731213401431/Put/seqid=0 2024-11-10T04:36:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T04:36:42,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741865_1041 (size=12151) 2024-11-10T04:36:43,315 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/5f07af19421e4976af81c7ee83a825d2 2024-11-10T04:36:43,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/bf6ae0199c7d4b588404f216aea4d92d is 50, key is test_row_0/C:col10/1731213401431/Put/seqid=0 2024-11-10T04:36:43,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741866_1042 (size=12151) 2024-11-10T04:36:43,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:43,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213463616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:43,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213463619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:43,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213463621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:43,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:43,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213463731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:43,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213463731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:43,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213463731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:43,748 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/bf6ae0199c7d4b588404f216aea4d92d 2024-11-10T04:36:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/5efb4edc2675426ea21add4c521a9265 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5efb4edc2675426ea21add4c521a9265 2024-11-10T04:36:43,768 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5efb4edc2675426ea21add4c521a9265, entries=150, sequenceid=157, filesize=11.9 K 2024-11-10T04:36:43,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/5f07af19421e4976af81c7ee83a825d2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5f07af19421e4976af81c7ee83a825d2 2024-11-10T04:36:43,777 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5f07af19421e4976af81c7ee83a825d2, entries=150, sequenceid=157, filesize=11.9 K 2024-11-10T04:36:43,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/bf6ae0199c7d4b588404f216aea4d92d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/bf6ae0199c7d4b588404f216aea4d92d 2024-11-10T04:36:43,787 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/bf6ae0199c7d4b588404f216aea4d92d, entries=150, sequenceid=157, filesize=11.9 K 2024-11-10T04:36:43,788 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6d2efd97fd4ccb14e877b384767c0ffc in 948ms, sequenceid=157, compaction requested=true 2024-11-10T04:36:43,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:43,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:43,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T04:36:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T04:36:43,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T04:36:43,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0420 sec 2024-11-10T04:36:43,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0530 sec 2024-11-10T04:36:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T04:36:43,892 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:43,892 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T04:36:43,897 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T04:36:43,898 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:43,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:43,944 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:43,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T04:36:43,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/b787537def8a4384991a0f59e1bcde96 is 50, key is test_row_0/A:col10/1731213403942/Put/seqid=0 2024-11-10T04:36:43,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741867_1043 (size=12151) 2024-11-10T04:36:43,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213463992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:43,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213463993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213463992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213463994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T04:36:44,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,000 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 6d2efd97fd4ccb14e877b384767c0ffc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213463996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T04:36:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:44,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213464102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213464111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213464111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213464111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T04:36:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T04:36:44,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213464312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213464321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213464322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213464322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T04:36:44,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/b787537def8a4384991a0f59e1bcde96 2024-11-10T04:36:44,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/9c0f896a61ef43e4b993f9fcaa3c25bd is 50, key is test_row_0/B:col10/1731213403942/Put/seqid=0 2024-11-10T04:36:44,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741868_1044 (size=12151) 2024-11-10T04:36:44,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T04:36:44,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:44,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T04:36:44,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213464621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213464632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213464632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213464632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:44,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T04:36:44,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:44,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/9c0f896a61ef43e4b993f9fcaa3c25bd 2024-11-10T04:36:44,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/8f49c9bf2df542428b85048b0ae0e985 is 50, key is test_row_0/C:col10/1731213403942/Put/seqid=0 2024-11-10T04:36:44,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T04:36:44,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:44,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:44,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741869_1045 (size=12151) 2024-11-10T04:36:44,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/8f49c9bf2df542428b85048b0ae0e985 2024-11-10T04:36:44,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/b787537def8a4384991a0f59e1bcde96 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b787537def8a4384991a0f59e1bcde96 2024-11-10T04:36:44,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b787537def8a4384991a0f59e1bcde96, entries=150, sequenceid=172, filesize=11.9 K 2024-11-10T04:36:44,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/9c0f896a61ef43e4b993f9fcaa3c25bd as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9c0f896a61ef43e4b993f9fcaa3c25bd 2024-11-10T04:36:44,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9c0f896a61ef43e4b993f9fcaa3c25bd, entries=150, sequenceid=172, filesize=11.9 K 2024-11-10T04:36:44,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/8f49c9bf2df542428b85048b0ae0e985 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/8f49c9bf2df542428b85048b0ae0e985 2024-11-10T04:36:44,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/8f49c9bf2df542428b85048b0ae0e985, entries=150, sequenceid=172, filesize=11.9 K 2024-11-10T04:36:44,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6d2efd97fd4ccb14e877b384767c0ffc in 950ms, sequenceid=172, compaction requested=true 2024-11-10T04:36:44,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:44,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:36:44,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:44,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:36:44,895 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:36:44,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:44,895 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:36:44,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:36:44,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:44,899 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:36:44,899 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:36:44,899 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/A is initiating minor compaction (all files) 2024-11-10T04:36:44,899 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/B is initiating minor compaction (all files) 2024-11-10T04:36:44,899 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/A in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,899 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/B in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,899 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/0e2742f6ec3d4bdc8638b2ecce918fff, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b66aaac446bf43edbfd353ea89a2be38, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5f07af19421e4976af81c7ee83a825d2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9c0f896a61ef43e4b993f9fcaa3c25bd] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=47.5 K 2024-11-10T04:36:44,899 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6655f07490f7479abde69388c75769d0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c5ad68ea14c94f4e822a7d3c86b8e5c3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5efb4edc2675426ea21add4c521a9265, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b787537def8a4384991a0f59e1bcde96] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=47.5 K 2024-11-10T04:36:44,900 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e2742f6ec3d4bdc8638b2ecce918fff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213400853 2024-11-10T04:36:44,901 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6655f07490f7479abde69388c75769d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213400853 2024-11-10T04:36:44,901 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b66aaac446bf43edbfd353ea89a2be38, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731213401016 2024-11-10T04:36:44,901 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ad68ea14c94f4e822a7d3c86b8e5c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731213401016 2024-11-10T04:36:44,902 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5efb4edc2675426ea21add4c521a9265, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731213401388 2024-11-10T04:36:44,902 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f07af19421e4976af81c7ee83a825d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731213401388 2024-11-10T04:36:44,903 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c0f896a61ef43e4b993f9fcaa3c25bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731213403604 2024-11-10T04:36:44,904 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b787537def8a4384991a0f59e1bcde96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731213403604 2024-11-10T04:36:44,931 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#B#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:44,932 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/5685bbdb433e478c96bd04f9058c0849 is 50, key is test_row_0/B:col10/1731213403942/Put/seqid=0 2024-11-10T04:36:44,943 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:44,944 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/6f8368ec87444933a823d5f4ab00bf75 is 50, key is test_row_0/A:col10/1731213403942/Put/seqid=0 2024-11-10T04:36:44,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741870_1046 (size=12527) 2024-11-10T04:36:44,963 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/5685bbdb433e478c96bd04f9058c0849 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5685bbdb433e478c96bd04f9058c0849 2024-11-10T04:36:44,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741871_1047 (size=12527) 2024-11-10T04:36:44,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T04:36:44,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,982 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T04:36:44,982 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/B of 6d2efd97fd4ccb14e877b384767c0ffc into 5685bbdb433e478c96bd04f9058c0849(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:44,982 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:44,982 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/B, priority=12, startTime=1731213404895; duration=0sec 2024-11-10T04:36:44,983 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:44,983 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:B 2024-11-10T04:36:44,983 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:36:44,987 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:36:44,987 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/C is initiating minor compaction (all files) 2024-11-10T04:36:44,987 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/C in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:44,987 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/51a89506e538477ab8e80b4c94b78bfb, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/ae926e56c92f46b48f3174ab9ae7257b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/bf6ae0199c7d4b588404f216aea4d92d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/8f49c9bf2df542428b85048b0ae0e985] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=47.5 K 2024-11-10T04:36:44,988 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a89506e538477ab8e80b4c94b78bfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213400853 2024-11-10T04:36:44,990 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ae926e56c92f46b48f3174ab9ae7257b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731213401016 2024-11-10T04:36:44,991 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting bf6ae0199c7d4b588404f216aea4d92d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731213401388 2024-11-10T04:36:44,992 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f49c9bf2df542428b85048b0ae0e985, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731213403604 2024-11-10T04:36:44,993 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/6f8368ec87444933a823d5f4ab00bf75 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6f8368ec87444933a823d5f4ab00bf75 2024-11-10T04:36:44,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/28e09bd3bdaf4de4ae875d2f2246e200 is 50, key is test_row_0/A:col10/1731213403989/Put/seqid=0 2024-11-10T04:36:45,004 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/A of 6d2efd97fd4ccb14e877b384767c0ffc into 6f8368ec87444933a823d5f4ab00bf75(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:45,004 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:45,004 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/A, priority=12, startTime=1731213404894; duration=0sec 2024-11-10T04:36:45,004 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:45,004 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:A 2024-11-10T04:36:45,019 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:45,020 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/b18f691db64f413c908244281192aa5b is 50, key is test_row_0/C:col10/1731213403942/Put/seqid=0 2024-11-10T04:36:45,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741872_1048 (size=12151) 2024-11-10T04:36:45,023 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/28e09bd3bdaf4de4ae875d2f2246e200 2024-11-10T04:36:45,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T04:36:45,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741873_1049 (size=12527) 2024-11-10T04:36:45,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/edaaf027ebe8476db81d73683dfcd036 is 50, key is test_row_0/B:col10/1731213403989/Put/seqid=0 2024-11-10T04:36:45,055 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/b18f691db64f413c908244281192aa5b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b18f691db64f413c908244281192aa5b 2024-11-10T04:36:45,065 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/C of 6d2efd97fd4ccb14e877b384767c0ffc into b18f691db64f413c908244281192aa5b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:45,065 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:45,065 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/C, priority=12, startTime=1731213404895; duration=0sec 2024-11-10T04:36:45,065 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:45,065 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:36:45,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741874_1050 (size=12151) 2024-11-10T04:36:45,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:45,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213465159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213465160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213465160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213465163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213465272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213465272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213465272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213465272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,475 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/edaaf027ebe8476db81d73683dfcd036 2024-11-10T04:36:45,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213465481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213465482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213465482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213465482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/4f83e7512d3e4d3abff909572fb30063 is 50, key is test_row_0/C:col10/1731213403989/Put/seqid=0 2024-11-10T04:36:45,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741875_1051 (size=12151) 2024-11-10T04:36:45,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213465792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213465792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213465792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213465801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:45,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:45,899 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/4f83e7512d3e4d3abff909572fb30063 2024-11-10T04:36:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/28e09bd3bdaf4de4ae875d2f2246e200 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/28e09bd3bdaf4de4ae875d2f2246e200 2024-11-10T04:36:45,918 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/28e09bd3bdaf4de4ae875d2f2246e200, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T04:36:45,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/edaaf027ebe8476db81d73683dfcd036 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/edaaf027ebe8476db81d73683dfcd036 2024-11-10T04:36:45,935 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/edaaf027ebe8476db81d73683dfcd036, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T04:36:45,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/4f83e7512d3e4d3abff909572fb30063 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/4f83e7512d3e4d3abff909572fb30063 2024-11-10T04:36:45,945 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/4f83e7512d3e4d3abff909572fb30063, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T04:36:45,946 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6d2efd97fd4ccb14e877b384767c0ffc in 964ms, sequenceid=194, compaction requested=false 2024-11-10T04:36:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T04:36:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T04:36:45,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T04:36:45,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0500 sec 2024-11-10T04:36:45,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.0600 sec 2024-11-10T04:36:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T04:36:46,041 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:46,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T04:36:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T04:36:46,045 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:46,046 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:46,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T04:36:46,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T04:36:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:46,199 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T04:36:46,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/b21d19292b3c437590d2723f37b60565 is 50, key is test_row_0/A:col10/1731213405158/Put/seqid=0 2024-11-10T04:36:46,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741876_1052 (size=12151) 2024-11-10T04:36:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:46,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213466333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213466335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213466340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213466340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T04:36:46,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213466452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213466452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213466452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213466452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,618 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/b21d19292b3c437590d2723f37b60565 2024-11-10T04:36:46,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/cf09cdd9038e4c89b03628e911f3929d is 50, key is test_row_0/B:col10/1731213405158/Put/seqid=0 2024-11-10T04:36:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741877_1053 (size=12151) 2024-11-10T04:36:46,637 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/cf09cdd9038e4c89b03628e911f3929d 2024-11-10T04:36:46,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/0aff9481ad614b66a75311f249af2512 is 50, key is test_row_0/C:col10/1731213405158/Put/seqid=0 2024-11-10T04:36:46,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741878_1054 (size=12151) 2024-11-10T04:36:46,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213466661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213466661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213466661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213466661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T04:36:46,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213466971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213466971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213466971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213466971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,053 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/0aff9481ad614b66a75311f249af2512 2024-11-10T04:36:47,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/b21d19292b3c437590d2723f37b60565 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b21d19292b3c437590d2723f37b60565 2024-11-10T04:36:47,069 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b21d19292b3c437590d2723f37b60565, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T04:36:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/cf09cdd9038e4c89b03628e911f3929d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cf09cdd9038e4c89b03628e911f3929d 2024-11-10T04:36:47,077 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cf09cdd9038e4c89b03628e911f3929d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T04:36:47,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/0aff9481ad614b66a75311f249af2512 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0aff9481ad614b66a75311f249af2512 2024-11-10T04:36:47,090 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0aff9481ad614b66a75311f249af2512, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T04:36:47,092 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6d2efd97fd4ccb14e877b384767c0ffc in 893ms, sequenceid=211, compaction requested=true 2024-11-10T04:36:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T04:36:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T04:36:47,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T04:36:47,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0480 sec 2024-11-10T04:36:47,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.0570 sec 2024-11-10T04:36:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T04:36:47,182 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:47,182 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T04:36:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T04:36:47,186 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:47,187 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:47,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T04:36:47,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T04:36:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:47,342 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T04:36:47,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/05f278b677d146f3af37ad087d0b05d1 is 50, key is test_row_0/A:col10/1731213406337/Put/seqid=0 2024-11-10T04:36:47,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741879_1055 (size=12151) 2024-11-10T04:36:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:47,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T04:36:47,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213467501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213467504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213467508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213467510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213467638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213467638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213467638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213467638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,753 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/05f278b677d146f3af37ad087d0b05d1 2024-11-10T04:36:47,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/581caffd42c84f7aa21f79c30aa26e30 is 50, key is test_row_0/B:col10/1731213406337/Put/seqid=0 2024-11-10T04:36:47,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741880_1056 (size=12151) 2024-11-10T04:36:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T04:36:47,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213467851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213467851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213467851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:47,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213467851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213468011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,013 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 6d2efd97fd4ccb14e877b384767c0ffc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:36:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213468161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213468161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213468162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213468162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,173 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/581caffd42c84f7aa21f79c30aa26e30 2024-11-10T04:36:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/66d978064dca4552bddefadf1e2570d4 is 50, key is test_row_0/C:col10/1731213406337/Put/seqid=0 2024-11-10T04:36:48,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741881_1057 (size=12151) 2024-11-10T04:36:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T04:36:48,592 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/66d978064dca4552bddefadf1e2570d4 2024-11-10T04:36:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/05f278b677d146f3af37ad087d0b05d1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/05f278b677d146f3af37ad087d0b05d1 2024-11-10T04:36:48,605 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/05f278b677d146f3af37ad087d0b05d1, entries=150, sequenceid=232, filesize=11.9 K 2024-11-10T04:36:48,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/581caffd42c84f7aa21f79c30aa26e30 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/581caffd42c84f7aa21f79c30aa26e30 2024-11-10T04:36:48,613 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/581caffd42c84f7aa21f79c30aa26e30, entries=150, sequenceid=232, filesize=11.9 K 2024-11-10T04:36:48,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/66d978064dca4552bddefadf1e2570d4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/66d978064dca4552bddefadf1e2570d4 2024-11-10T04:36:48,620 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/66d978064dca4552bddefadf1e2570d4, entries=150, sequenceid=232, filesize=11.9 K 2024-11-10T04:36:48,621 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6d2efd97fd4ccb14e877b384767c0ffc in 1280ms, sequenceid=232, compaction requested=true 2024-11-10T04:36:48,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:48,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:48,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T04:36:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T04:36:48,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T04:36:48,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4360 sec 2024-11-10T04:36:48,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4450 sec 2024-11-10T04:36:48,676 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:48,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T04:36:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:48,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/f7c8b0653653411c82412200c51a9fc5 is 50, key is test_row_0/A:col10/1731213407491/Put/seqid=0 2024-11-10T04:36:48,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741882_1058 (size=12151) 2024-11-10T04:36:48,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/f7c8b0653653411c82412200c51a9fc5 2024-11-10T04:36:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213468700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213468702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213468704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213468705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/1bdf30c4e3ec4d848a621bddb9950ad6 is 50, key is test_row_0/B:col10/1731213407491/Put/seqid=0 2024-11-10T04:36:48,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741883_1059 (size=12151) 2024-11-10T04:36:48,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213468812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213468812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213468812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213468822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213469021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213469021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213469021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213469031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/1bdf30c4e3ec4d848a621bddb9950ad6 2024-11-10T04:36:49,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/69bbd2cb3ecf4447986a05a19e8a1f6e is 50, key is test_row_0/C:col10/1731213407491/Put/seqid=0 2024-11-10T04:36:49,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741884_1060 (size=12151) 2024-11-10T04:36:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T04:36:49,332 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213469331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,332 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213469332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-10T04:36:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213469332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T04:36:49,335 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:49,336 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:49,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213469341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T04:36:49,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T04:36:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:49,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:49,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/69bbd2cb3ecf4447986a05a19e8a1f6e 2024-11-10T04:36:49,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/f7c8b0653653411c82412200c51a9fc5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f7c8b0653653411c82412200c51a9fc5 2024-11-10T04:36:49,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f7c8b0653653411c82412200c51a9fc5, entries=150, sequenceid=250, filesize=11.9 K 2024-11-10T04:36:49,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/1bdf30c4e3ec4d848a621bddb9950ad6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/1bdf30c4e3ec4d848a621bddb9950ad6 2024-11-10T04:36:49,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/1bdf30c4e3ec4d848a621bddb9950ad6, entries=150, sequenceid=250, filesize=11.9 K 2024-11-10T04:36:49,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/69bbd2cb3ecf4447986a05a19e8a1f6e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/69bbd2cb3ecf4447986a05a19e8a1f6e 2024-11-10T04:36:49,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/69bbd2cb3ecf4447986a05a19e8a1f6e, entries=150, sequenceid=250, filesize=11.9 K 2024-11-10T04:36:49,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 6d2efd97fd4ccb14e877b384767c0ffc in 888ms, sequenceid=250, compaction requested=true 2024-11-10T04:36:49,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:49,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:36:49,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:49,564 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T04:36:49,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:36:49,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:49,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:36:49,564 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T04:36:49,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:49,567 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T04:36:49,567 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/A is initiating minor compaction (all files) 2024-11-10T04:36:49,567 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T04:36:49,567 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/A in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:49,567 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/B is initiating minor compaction (all files) 2024-11-10T04:36:49,567 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6f8368ec87444933a823d5f4ab00bf75, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/28e09bd3bdaf4de4ae875d2f2246e200, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b21d19292b3c437590d2723f37b60565, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/05f278b677d146f3af37ad087d0b05d1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f7c8b0653653411c82412200c51a9fc5] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=59.7 K 2024-11-10T04:36:49,567 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/B in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:49,567 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5685bbdb433e478c96bd04f9058c0849, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/edaaf027ebe8476db81d73683dfcd036, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cf09cdd9038e4c89b03628e911f3929d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/581caffd42c84f7aa21f79c30aa26e30, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/1bdf30c4e3ec4d848a621bddb9950ad6] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=59.7 K 2024-11-10T04:36:49,568 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5685bbdb433e478c96bd04f9058c0849, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731213403604 2024-11-10T04:36:49,568 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f8368ec87444933a823d5f4ab00bf75, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731213403604 2024-11-10T04:36:49,568 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting edaaf027ebe8476db81d73683dfcd036, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731213403985 2024-11-10T04:36:49,569 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28e09bd3bdaf4de4ae875d2f2246e200, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731213403985 2024-11-10T04:36:49,569 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cf09cdd9038e4c89b03628e911f3929d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731213405156 2024-11-10T04:36:49,569 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b21d19292b3c437590d2723f37b60565, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731213405156 2024-11-10T04:36:49,569 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 581caffd42c84f7aa21f79c30aa26e30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731213406327 2024-11-10T04:36:49,570 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05f278b677d146f3af37ad087d0b05d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731213406327 2024-11-10T04:36:49,570 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bdf30c4e3ec4d848a621bddb9950ad6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731213407491 2024-11-10T04:36:49,570 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7c8b0653653411c82412200c51a9fc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731213407491 2024-11-10T04:36:49,587 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:49,587 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/bcafc56cd95b47178a3a487e0b055736 is 50, key is test_row_0/B:col10/1731213407491/Put/seqid=0 2024-11-10T04:36:49,589 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:49,590 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/e8c939fc17eb42f396fdf9e9c23ac42e is 50, key is test_row_0/A:col10/1731213407491/Put/seqid=0 2024-11-10T04:36:49,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741885_1061 (size=12697) 2024-11-10T04:36:49,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741886_1062 (size=12697) 2024-11-10T04:36:49,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T04:36:49,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:49,644 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-10T04:36:49,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/57a76a743c0c4968b9b4e513cd898a59 is 50, key is test_row_0/A:col10/1731213408700/Put/seqid=0 2024-11-10T04:36:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T04:36:49,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741887_1063 (size=12301) 2024-11-10T04:36:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:49,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:49,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213469865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213469866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213469867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213469869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T04:36:49,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213469981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213469981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213469981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:49,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213469981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:50,006 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/bcafc56cd95b47178a3a487e0b055736 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/bcafc56cd95b47178a3a487e0b055736 2024-11-10T04:36:50,006 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/e8c939fc17eb42f396fdf9e9c23ac42e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/e8c939fc17eb42f396fdf9e9c23ac42e 2024-11-10T04:36:50,015 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/A of 6d2efd97fd4ccb14e877b384767c0ffc into e8c939fc17eb42f396fdf9e9c23ac42e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:50,015 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:50,015 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/A, priority=11, startTime=1731213409564; duration=0sec 2024-11-10T04:36:50,016 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/B of 6d2efd97fd4ccb14e877b384767c0ffc into bcafc56cd95b47178a3a487e0b055736(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:50,016 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:50,016 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/B, priority=11, startTime=1731213409564; duration=0sec 2024-11-10T04:36:50,016 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:50,016 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:B 2024-11-10T04:36:50,016 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T04:36:50,017 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:50,017 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:A 2024-11-10T04:36:50,019 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T04:36:50,019 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/C is initiating minor compaction (all files) 2024-11-10T04:36:50,019 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/C in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:50,019 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b18f691db64f413c908244281192aa5b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/4f83e7512d3e4d3abff909572fb30063, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0aff9481ad614b66a75311f249af2512, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/66d978064dca4552bddefadf1e2570d4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/69bbd2cb3ecf4447986a05a19e8a1f6e] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=59.7 K 2024-11-10T04:36:50,020 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b18f691db64f413c908244281192aa5b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731213403604 2024-11-10T04:36:50,020 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f83e7512d3e4d3abff909572fb30063, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731213403985 2024-11-10T04:36:50,020 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aff9481ad614b66a75311f249af2512, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731213405156 2024-11-10T04:36:50,021 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d978064dca4552bddefadf1e2570d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731213406327 2024-11-10T04:36:50,021 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 69bbd2cb3ecf4447986a05a19e8a1f6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731213407491 2024-11-10T04:36:50,036 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#C#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:50,037 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/adde20b42a274d5293a41963b836bb11 is 50, key is test_row_0/C:col10/1731213407491/Put/seqid=0 2024-11-10T04:36:50,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741888_1064 (size=12697) 2024-11-10T04:36:50,056 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/57a76a743c0c4968b9b4e513cd898a59 2024-11-10T04:36:50,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/dbb86552188c4ec5aedb6f226e543b42 is 50, key is test_row_0/B:col10/1731213408700/Put/seqid=0 2024-11-10T04:36:50,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741889_1065 (size=12301) 2024-11-10T04:36:50,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213470191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:50,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213470191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213470191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213470191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:50,450 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/adde20b42a274d5293a41963b836bb11 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/adde20b42a274d5293a41963b836bb11 2024-11-10T04:36:50,458 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/C of 6d2efd97fd4ccb14e877b384767c0ffc into adde20b42a274d5293a41963b836bb11(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:50,458 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:50,458 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/C, priority=11, startTime=1731213409564; duration=0sec 2024-11-10T04:36:50,458 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:50,458 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:36:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T04:36:50,472 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/dbb86552188c4ec5aedb6f226e543b42 2024-11-10T04:36:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/1d18535a037d48349d7a224669d66b0e is 50, key is test_row_0/C:col10/1731213408700/Put/seqid=0 2024-11-10T04:36:50,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741890_1066 (size=12301) 2024-11-10T04:36:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213470501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213470501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:50,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213470501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213470502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:50,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:50,889 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/1d18535a037d48349d7a224669d66b0e 2024-11-10T04:36:50,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/57a76a743c0c4968b9b4e513cd898a59 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57a76a743c0c4968b9b4e513cd898a59 2024-11-10T04:36:50,902 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57a76a743c0c4968b9b4e513cd898a59, entries=150, sequenceid=268, filesize=12.0 K 2024-11-10T04:36:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/dbb86552188c4ec5aedb6f226e543b42 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/dbb86552188c4ec5aedb6f226e543b42 2024-11-10T04:36:50,910 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/dbb86552188c4ec5aedb6f226e543b42, entries=150, sequenceid=268, filesize=12.0 K 2024-11-10T04:36:50,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/1d18535a037d48349d7a224669d66b0e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1d18535a037d48349d7a224669d66b0e 2024-11-10T04:36:50,916 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1d18535a037d48349d7a224669d66b0e, entries=150, sequenceid=268, filesize=12.0 K 2024-11-10T04:36:50,918 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 6d2efd97fd4ccb14e877b384767c0ffc in 1274ms, sequenceid=268, compaction requested=false 2024-11-10T04:36:50,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:50,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:50,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-10T04:36:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-10T04:36:50,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T04:36:50,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5840 sec 2024-11-10T04:36:50,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5910 sec 2024-11-10T04:36:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:51,026 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:51,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T04:36:51,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8c5c6a4440df4b0d83f4cb4266f5abe9 is 50, key is test_row_0/A:col10/1731213411024/Put/seqid=0 2024-11-10T04:36:51,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741891_1067 (size=14741) 2024-11-10T04:36:51,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8c5c6a4440df4b0d83f4cb4266f5abe9 2024-11-10T04:36:51,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213471044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/c23c4d4860df4bb7acf4b5316db40a09 is 50, key is test_row_0/B:col10/1731213411024/Put/seqid=0 2024-11-10T04:36:51,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213471054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213471054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213471054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741892_1068 (size=12301) 2024-11-10T04:36:51,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/c23c4d4860df4bb7acf4b5316db40a09 2024-11-10T04:36:51,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/a585506c2d514d5d86cf5071bca0b8e4 is 50, key is test_row_0/C:col10/1731213411024/Put/seqid=0 2024-11-10T04:36:51,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741893_1069 (size=12301) 2024-11-10T04:36:51,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213471161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213471172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213471172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213471172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213471372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213471381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213471381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213471381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T04:36:51,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/a585506c2d514d5d86cf5071bca0b8e4 2024-11-10T04:36:51,492 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:51,492 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-10T04:36:51,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8c5c6a4440df4b0d83f4cb4266f5abe9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8c5c6a4440df4b0d83f4cb4266f5abe9 2024-11-10T04:36:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T04:36:51,506 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:51,507 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:51,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:51,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8c5c6a4440df4b0d83f4cb4266f5abe9, entries=200, sequenceid=292, filesize=14.4 K 2024-11-10T04:36:51,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/c23c4d4860df4bb7acf4b5316db40a09 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c23c4d4860df4bb7acf4b5316db40a09 2024-11-10T04:36:51,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c23c4d4860df4bb7acf4b5316db40a09, entries=150, sequenceid=292, filesize=12.0 K 2024-11-10T04:36:51,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/a585506c2d514d5d86cf5071bca0b8e4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a585506c2d514d5d86cf5071bca0b8e4 2024-11-10T04:36:51,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a585506c2d514d5d86cf5071bca0b8e4, entries=150, sequenceid=292, filesize=12.0 K 2024-11-10T04:36:51,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 6d2efd97fd4ccb14e877b384767c0ffc in 498ms, sequenceid=292, compaction requested=true 2024-11-10T04:36:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:36:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:51,525 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:51,525 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:36:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:36:51,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:51,527 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:51,527 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/A is initiating minor compaction (all files) 2024-11-10T04:36:51,527 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/A in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:51,527 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/e8c939fc17eb42f396fdf9e9c23ac42e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57a76a743c0c4968b9b4e513cd898a59, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8c5c6a4440df4b0d83f4cb4266f5abe9] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=38.8 K 2024-11-10T04:36:51,528 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c939fc17eb42f396fdf9e9c23ac42e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731213407491 2024-11-10T04:36:51,528 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:51,528 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/B is initiating minor compaction (all files) 2024-11-10T04:36:51,528 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/B in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:51,528 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/bcafc56cd95b47178a3a487e0b055736, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/dbb86552188c4ec5aedb6f226e543b42, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c23c4d4860df4bb7acf4b5316db40a09] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=36.4 K 2024-11-10T04:36:51,528 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a76a743c0c4968b9b4e513cd898a59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731213408698 2024-11-10T04:36:51,529 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c5c6a4440df4b0d83f4cb4266f5abe9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731213409862 2024-11-10T04:36:51,529 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting bcafc56cd95b47178a3a487e0b055736, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731213407491 2024-11-10T04:36:51,531 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting dbb86552188c4ec5aedb6f226e543b42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731213408698 2024-11-10T04:36:51,531 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c23c4d4860df4bb7acf4b5316db40a09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731213409862 2024-11-10T04:36:51,542 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#A#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:51,542 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/4adc06991e784982a95e157857f5da7d is 50, key is test_row_0/A:col10/1731213411024/Put/seqid=0 2024-11-10T04:36:51,545 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:51,546 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/142e6dd96fe945d68aafe39988f53325 is 50, key is test_row_0/B:col10/1731213411024/Put/seqid=0 2024-11-10T04:36:51,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741894_1070 (size=12949) 2024-11-10T04:36:51,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741895_1071 (size=12949) 2024-11-10T04:36:51,580 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/142e6dd96fe945d68aafe39988f53325 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/142e6dd96fe945d68aafe39988f53325 2024-11-10T04:36:51,589 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/B of 6d2efd97fd4ccb14e877b384767c0ffc into 142e6dd96fe945d68aafe39988f53325(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:51,589 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:51,590 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/B, priority=13, startTime=1731213411525; duration=0sec 2024-11-10T04:36:51,592 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:51,592 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:B 2024-11-10T04:36:51,592 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:51,594 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:51,594 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/C is initiating minor compaction (all files) 2024-11-10T04:36:51,594 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/C in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:51,594 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/adde20b42a274d5293a41963b836bb11, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1d18535a037d48349d7a224669d66b0e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a585506c2d514d5d86cf5071bca0b8e4] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=36.4 K 2024-11-10T04:36:51,595 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting adde20b42a274d5293a41963b836bb11, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731213407491 2024-11-10T04:36:51,595 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d18535a037d48349d7a224669d66b0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731213408698 2024-11-10T04:36:51,596 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a585506c2d514d5d86cf5071bca0b8e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731213409862 2024-11-10T04:36:51,609 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#C#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:51,610 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/142ba19a55f545518f0b58ed72b2a661 is 50, key is test_row_0/C:col10/1731213411024/Put/seqid=0 2024-11-10T04:36:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T04:36:51,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741896_1072 (size=12949) 2024-11-10T04:36:51,635 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/142ba19a55f545518f0b58ed72b2a661 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/142ba19a55f545518f0b58ed72b2a661 2024-11-10T04:36:51,645 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/C of 6d2efd97fd4ccb14e877b384767c0ffc into 142ba19a55f545518f0b58ed72b2a661(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:51,645 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:51,645 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/C, priority=13, startTime=1731213411525; duration=0sec 2024-11-10T04:36:51,646 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:51,646 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:36:51,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T04:36:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:51,661 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T04:36:51,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/57090a8fef944f5eacaba22cb1265dfe is 50, key is test_row_0/A:col10/1731213411038/Put/seqid=0 2024-11-10T04:36:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:51,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:51,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741897_1073 (size=12301) 2024-11-10T04:36:51,688 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/57090a8fef944f5eacaba22cb1265dfe 2024-11-10T04:36:51,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/b8f2bb7d4cfd4d38b5dc3c6adb24a5da is 50, key is test_row_0/B:col10/1731213411038/Put/seqid=0 2024-11-10T04:36:51,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741898_1074 (size=12301) 2024-11-10T04:36:51,717 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/b8f2bb7d4cfd4d38b5dc3c6adb24a5da 2024-11-10T04:36:51,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/210270c117bd46ad90710a3b376cb241 is 50, key is test_row_0/C:col10/1731213411038/Put/seqid=0 2024-11-10T04:36:51,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741899_1075 (size=12301) 2024-11-10T04:36:51,738 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/210270c117bd46ad90710a3b376cb241 2024-11-10T04:36:51,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/57090a8fef944f5eacaba22cb1265dfe as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57090a8fef944f5eacaba22cb1265dfe 2024-11-10T04:36:51,748 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57090a8fef944f5eacaba22cb1265dfe, entries=150, sequenceid=310, filesize=12.0 K 2024-11-10T04:36:51,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/b8f2bb7d4cfd4d38b5dc3c6adb24a5da as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b8f2bb7d4cfd4d38b5dc3c6adb24a5da 2024-11-10T04:36:51,755 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b8f2bb7d4cfd4d38b5dc3c6adb24a5da, entries=150, sequenceid=310, filesize=12.0 K 2024-11-10T04:36:51,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/210270c117bd46ad90710a3b376cb241 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/210270c117bd46ad90710a3b376cb241 2024-11-10T04:36:51,762 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/210270c117bd46ad90710a3b376cb241, entries=150, sequenceid=310, filesize=12.0 K 2024-11-10T04:36:51,769 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=67.09 KB/68700 for 6d2efd97fd4ccb14e877b384767c0ffc in 108ms, sequenceid=310, compaction requested=false 2024-11-10T04:36:51,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:51,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:51,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-10T04:36:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:51,770 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:51,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T04:36:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-10T04:36:51,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/eed3ca8a10264a14aa29ebb72b2332e7 is 50, key is test_row_0/A:col10/1731213411709/Put/seqid=0 2024-11-10T04:36:51,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T04:36:51,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 265 msec 2024-11-10T04:36:51,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 285 msec 2024-11-10T04:36:51,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741900_1076 (size=12301) 2024-11-10T04:36:51,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/eed3ca8a10264a14aa29ebb72b2332e7 2024-11-10T04:36:51,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/a88e6f9f519a45a89a5ea50103e935cb is 50, key is test_row_0/B:col10/1731213411709/Put/seqid=0 2024-11-10T04:36:51,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741901_1077 (size=12301) 2024-11-10T04:36:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T04:36:51,822 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:51,822 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-10T04:36:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T04:36:51,824 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:51,825 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:51,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:51,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213471846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213471848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213471849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213471853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T04:36:51,963 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/4adc06991e784982a95e157857f5da7d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/4adc06991e784982a95e157857f5da7d 2024-11-10T04:36:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213471961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213471961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213471961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213471962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:51,973 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/A of 6d2efd97fd4ccb14e877b384767c0ffc into 4adc06991e784982a95e157857f5da7d(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:51,973 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:51,973 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/A, priority=13, startTime=1731213411525; duration=0sec 2024-11-10T04:36:51,973 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:51,973 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:A 2024-11-10T04:36:51,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T04:36:51,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:51,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:51,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:51,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T04:36:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T04:36:52,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213472171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:52,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213472172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:52,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213472172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213472172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:52,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/a88e6f9f519a45a89a5ea50103e935cb 2024-11-10T04:36:52,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/019f901b37db493e87864d7e2b7762d4 is 50, key is test_row_0/C:col10/1731213411709/Put/seqid=0 2024-11-10T04:36:52,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741902_1078 (size=12301) 2024-11-10T04:36:52,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T04:36:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:52,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T04:36:52,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:52,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T04:36:52,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213472482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213472482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213472482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213472482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:52,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:52,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T04:36:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:52,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:52,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/019f901b37db493e87864d7e2b7762d4 2024-11-10T04:36:52,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/eed3ca8a10264a14aa29ebb72b2332e7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/eed3ca8a10264a14aa29ebb72b2332e7 2024-11-10T04:36:52,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/eed3ca8a10264a14aa29ebb72b2332e7, entries=150, sequenceid=326, filesize=12.0 K 2024-11-10T04:36:52,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/a88e6f9f519a45a89a5ea50103e935cb as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a88e6f9f519a45a89a5ea50103e935cb 2024-11-10T04:36:52,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a88e6f9f519a45a89a5ea50103e935cb, entries=150, sequenceid=326, filesize=12.0 K 2024-11-10T04:36:52,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/019f901b37db493e87864d7e2b7762d4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/019f901b37db493e87864d7e2b7762d4 2024-11-10T04:36:52,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/019f901b37db493e87864d7e2b7762d4, entries=150, sequenceid=326, filesize=12.0 K 2024-11-10T04:36:52,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6d2efd97fd4ccb14e877b384767c0ffc in 899ms, sequenceid=326, compaction requested=true 2024-11-10T04:36:52,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:36:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:52,670 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:52,670 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:36:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:36:52,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:52,671 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:52,671 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:52,671 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/B is initiating minor compaction (all files) 2024-11-10T04:36:52,671 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/A is initiating minor compaction (all files) 2024-11-10T04:36:52,671 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/B in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,671 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/A in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,672 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/142e6dd96fe945d68aafe39988f53325, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b8f2bb7d4cfd4d38b5dc3c6adb24a5da, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a88e6f9f519a45a89a5ea50103e935cb] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=36.7 K 2024-11-10T04:36:52,672 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/4adc06991e784982a95e157857f5da7d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57090a8fef944f5eacaba22cb1265dfe, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/eed3ca8a10264a14aa29ebb72b2332e7] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=36.7 K 2024-11-10T04:36:52,672 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 142e6dd96fe945d68aafe39988f53325, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731213409862 2024-11-10T04:36:52,672 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4adc06991e784982a95e157857f5da7d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731213409862 2024-11-10T04:36:52,673 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f2bb7d4cfd4d38b5dc3c6adb24a5da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731213411038 2024-11-10T04:36:52,673 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57090a8fef944f5eacaba22cb1265dfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731213411038 2024-11-10T04:36:52,673 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a88e6f9f519a45a89a5ea50103e935cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731213411709 2024-11-10T04:36:52,674 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed3ca8a10264a14aa29ebb72b2332e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731213411709 2024-11-10T04:36:52,686 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:52,686 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:52,686 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/6820a58986094f27a6e31a8b80a210bc is 50, key is test_row_0/A:col10/1731213411709/Put/seqid=0 2024-11-10T04:36:52,687 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/e0367834a93244ca85683ac3484efac2 is 50, key is test_row_0/B:col10/1731213411709/Put/seqid=0 2024-11-10T04:36:52,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741904_1080 (size=13051) 2024-11-10T04:36:52,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741903_1079 (size=13051) 2024-11-10T04:36:52,703 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/6820a58986094f27a6e31a8b80a210bc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6820a58986094f27a6e31a8b80a210bc 2024-11-10T04:36:52,713 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/A of 6d2efd97fd4ccb14e877b384767c0ffc into 6820a58986094f27a6e31a8b80a210bc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:52,713 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:52,713 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/A, priority=13, startTime=1731213412669; duration=0sec 2024-11-10T04:36:52,713 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:52,713 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:A 2024-11-10T04:36:52,713 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:52,715 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:52,715 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/C is initiating minor compaction (all files) 2024-11-10T04:36:52,715 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/C in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,715 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/142ba19a55f545518f0b58ed72b2a661, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/210270c117bd46ad90710a3b376cb241, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/019f901b37db493e87864d7e2b7762d4] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=36.7 K 2024-11-10T04:36:52,716 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 142ba19a55f545518f0b58ed72b2a661, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731213409862 2024-11-10T04:36:52,716 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 210270c117bd46ad90710a3b376cb241, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731213411038 2024-11-10T04:36:52,716 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019f901b37db493e87864d7e2b7762d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731213411709 2024-11-10T04:36:52,724 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:52,725 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/f595c7d6f54040bfbd45cd1795c71164 is 50, key is test_row_0/C:col10/1731213411709/Put/seqid=0 2024-11-10T04:36:52,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741905_1081 (size=13051) 2024-11-10T04:36:52,743 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/f595c7d6f54040bfbd45cd1795c71164 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/f595c7d6f54040bfbd45cd1795c71164 2024-11-10T04:36:52,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T04:36:52,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:52,748 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T04:36:52,750 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/C of 6d2efd97fd4ccb14e877b384767c0ffc into f595c7d6f54040bfbd45cd1795c71164(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:52,751 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:52,751 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/C, priority=13, startTime=1731213412670; duration=0sec 2024-11-10T04:36:52,751 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:52,751 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:36:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/419eebbf83d1409eb6b6a5621d098925 is 50, key is test_row_0/A:col10/1731213411842/Put/seqid=0 2024-11-10T04:36:52,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741906_1082 (size=12301) 2024-11-10T04:36:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T04:36:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:52,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:53,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213473009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213473012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213473014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213473014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,101 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/e0367834a93244ca85683ac3484efac2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/e0367834a93244ca85683ac3484efac2 2024-11-10T04:36:53,108 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/B of 6d2efd97fd4ccb14e877b384767c0ffc into e0367834a93244ca85683ac3484efac2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:53,108 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:53,108 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/B, priority=13, startTime=1731213412670; duration=0sec 2024-11-10T04:36:53,108 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:53,108 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:B 2024-11-10T04:36:53,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213473121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213473121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213473132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213473132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,165 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/419eebbf83d1409eb6b6a5621d098925 2024-11-10T04:36:53,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/9a22c2773ae74266a13ee66bbe973ee2 is 50, key is test_row_0/B:col10/1731213411842/Put/seqid=0 2024-11-10T04:36:53,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741907_1083 (size=12301) 2024-11-10T04:36:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213473331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213473331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213473341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213473341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,580 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/9a22c2773ae74266a13ee66bbe973ee2 2024-11-10T04:36:53,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/0e2c588a3c2a4442a1c357b1a5decc8f is 50, key is test_row_0/C:col10/1731213411842/Put/seqid=0 2024-11-10T04:36:53,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741908_1084 (size=12301) 2024-11-10T04:36:53,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213473641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213473641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213473652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213473652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T04:36:54,001 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/0e2c588a3c2a4442a1c357b1a5decc8f 2024-11-10T04:36:54,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/419eebbf83d1409eb6b6a5621d098925 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/419eebbf83d1409eb6b6a5621d098925 2024-11-10T04:36:54,015 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/419eebbf83d1409eb6b6a5621d098925, entries=150, sequenceid=348, filesize=12.0 K 2024-11-10T04:36:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/9a22c2773ae74266a13ee66bbe973ee2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9a22c2773ae74266a13ee66bbe973ee2 2024-11-10T04:36:54,021 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9a22c2773ae74266a13ee66bbe973ee2, entries=150, sequenceid=348, filesize=12.0 K 2024-11-10T04:36:54,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/0e2c588a3c2a4442a1c357b1a5decc8f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0e2c588a3c2a4442a1c357b1a5decc8f 2024-11-10T04:36:54,027 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0e2c588a3c2a4442a1c357b1a5decc8f, entries=150, sequenceid=348, filesize=12.0 K 2024-11-10T04:36:54,031 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 6d2efd97fd4ccb14e877b384767c0ffc in 1282ms, sequenceid=348, compaction requested=false 2024-11-10T04:36:54,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:54,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:54,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-10T04:36:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-10T04:36:54,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-10T04:36:54,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2070 sec 2024-11-10T04:36:54,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.2140 sec 2024-11-10T04:36:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:54,153 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:54,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-10T04:36:54,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/432fdf1592274ad5b5de671f68f33142 is 50, key is test_row_0/A:col10/1731213414152/Put/seqid=0 2024-11-10T04:36:54,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741909_1085 (size=12301) 2024-11-10T04:36:54,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/432fdf1592274ad5b5de671f68f33142 2024-11-10T04:36:54,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213474174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213474177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213474181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213474181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/91992f76c8fe4d9ba1054d983431af8c is 50, key is test_row_0/B:col10/1731213414152/Put/seqid=0 2024-11-10T04:36:54,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741910_1086 (size=12301) 2024-11-10T04:36:54,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213474291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213474291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213474292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213474292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213474502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213474502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213474502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213474502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/91992f76c8fe4d9ba1054d983431af8c 2024-11-10T04:36:54,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/439e96cbc2664700b5c55ee22e6066ca is 50, key is test_row_0/C:col10/1731213414152/Put/seqid=0 2024-11-10T04:36:54,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741911_1087 (size=12301) 2024-11-10T04:36:54,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213474811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213474812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213474812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213474812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/439e96cbc2664700b5c55ee22e6066ca 2024-11-10T04:36:55,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/432fdf1592274ad5b5de671f68f33142 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/432fdf1592274ad5b5de671f68f33142 2024-11-10T04:36:55,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/432fdf1592274ad5b5de671f68f33142, entries=150, sequenceid=369, filesize=12.0 K 2024-11-10T04:36:55,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/91992f76c8fe4d9ba1054d983431af8c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/91992f76c8fe4d9ba1054d983431af8c 2024-11-10T04:36:55,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/91992f76c8fe4d9ba1054d983431af8c, entries=150, sequenceid=369, filesize=12.0 K 2024-11-10T04:36:55,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/439e96cbc2664700b5c55ee22e6066ca as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/439e96cbc2664700b5c55ee22e6066ca 2024-11-10T04:36:55,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/439e96cbc2664700b5c55ee22e6066ca, entries=150, sequenceid=369, filesize=12.0 K 2024-11-10T04:36:55,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 6d2efd97fd4ccb14e877b384767c0ffc in 898ms, sequenceid=369, compaction requested=true 2024-11-10T04:36:55,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:55,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:36:55,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:55,051 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:36:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:55,052 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:36:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:55,053 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:55,053 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/A is initiating minor compaction (all files) 2024-11-10T04:36:55,053 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/A in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:55,053 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6820a58986094f27a6e31a8b80a210bc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/419eebbf83d1409eb6b6a5621d098925, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/432fdf1592274ad5b5de671f68f33142] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=36.8 K 2024-11-10T04:36:55,054 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:55,054 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/B is initiating minor compaction (all files) 2024-11-10T04:36:55,054 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/B in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:55,054 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/e0367834a93244ca85683ac3484efac2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9a22c2773ae74266a13ee66bbe973ee2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/91992f76c8fe4d9ba1054d983431af8c] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=36.8 K 2024-11-10T04:36:55,054 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6820a58986094f27a6e31a8b80a210bc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731213411709 2024-11-10T04:36:55,055 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e0367834a93244ca85683ac3484efac2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731213411709 2024-11-10T04:36:55,055 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 419eebbf83d1409eb6b6a5621d098925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731213411842 2024-11-10T04:36:55,055 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 432fdf1592274ad5b5de671f68f33142, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731213413001 2024-11-10T04:36:55,055 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a22c2773ae74266a13ee66bbe973ee2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731213411842 2024-11-10T04:36:55,056 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 91992f76c8fe4d9ba1054d983431af8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731213413001 2024-11-10T04:36:55,065 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:55,066 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/213372a5cf1a4484b5f68db6e4f4d98a is 50, key is test_row_0/A:col10/1731213414152/Put/seqid=0 2024-11-10T04:36:55,067 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:55,068 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/a45b3fefe38b44b086453a66198090f6 is 50, key is test_row_0/B:col10/1731213414152/Put/seqid=0 2024-11-10T04:36:55,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741912_1088 (size=13153) 2024-11-10T04:36:55,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741913_1089 (size=13153) 2024-11-10T04:36:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:55,325 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:55,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-10T04:36:55,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/b07fdd3d329a4f8f8b6aa8093a84822b is 50, key is test_row_0/A:col10/1731213414164/Put/seqid=0 2024-11-10T04:36:55,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741914_1090 (size=12301) 2024-11-10T04:36:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213475341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213475342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213475343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213475343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213475451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213475451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213475451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213475452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,483 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/213372a5cf1a4484b5f68db6e4f4d98a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/213372a5cf1a4484b5f68db6e4f4d98a 2024-11-10T04:36:55,490 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/A of 6d2efd97fd4ccb14e877b384767c0ffc into 213372a5cf1a4484b5f68db6e4f4d98a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:55,490 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:55,490 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/A, priority=13, startTime=1731213415051; duration=0sec 2024-11-10T04:36:55,490 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:55,490 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:A 2024-11-10T04:36:55,490 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:36:55,497 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:36:55,497 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/C is initiating minor compaction (all files) 2024-11-10T04:36:55,497 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/C in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:55,497 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/a45b3fefe38b44b086453a66198090f6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a45b3fefe38b44b086453a66198090f6 2024-11-10T04:36:55,497 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/f595c7d6f54040bfbd45cd1795c71164, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0e2c588a3c2a4442a1c357b1a5decc8f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/439e96cbc2664700b5c55ee22e6066ca] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=36.8 K 2024-11-10T04:36:55,497 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f595c7d6f54040bfbd45cd1795c71164, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731213411709 2024-11-10T04:36:55,498 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e2c588a3c2a4442a1c357b1a5decc8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731213411842 2024-11-10T04:36:55,499 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 439e96cbc2664700b5c55ee22e6066ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731213413001 2024-11-10T04:36:55,504 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/B of 6d2efd97fd4ccb14e877b384767c0ffc into a45b3fefe38b44b086453a66198090f6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:55,504 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:55,504 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/B, priority=13, startTime=1731213415051; duration=0sec 2024-11-10T04:36:55,504 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:55,504 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:B 2024-11-10T04:36:55,508 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:55,509 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/5ca2b073828f49d6a2e676c03efb7a6f is 50, key is test_row_0/C:col10/1731213414152/Put/seqid=0 2024-11-10T04:36:55,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741915_1091 (size=13153) 2024-11-10T04:36:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213475661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213475661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213475662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213475662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/b07fdd3d329a4f8f8b6aa8093a84822b 2024-11-10T04:36:55,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/7e6542541a134c38a247346fc12bf634 is 50, key is test_row_0/B:col10/1731213414164/Put/seqid=0 2024-11-10T04:36:55,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741916_1092 (size=12301) 2024-11-10T04:36:55,920 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/5ca2b073828f49d6a2e676c03efb7a6f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ca2b073828f49d6a2e676c03efb7a6f 2024-11-10T04:36:55,927 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/C of 6d2efd97fd4ccb14e877b384767c0ffc into 5ca2b073828f49d6a2e676c03efb7a6f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:55,927 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:55,927 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/C, priority=13, startTime=1731213415052; duration=0sec 2024-11-10T04:36:55,928 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:55,928 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:36:55,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213475971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213475971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213475971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213475971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T04:36:55,982 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:55,982 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-10T04:36:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T04:36:55,985 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:55,985 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:55,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T04:36:56,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T04:36:56,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:56,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:56,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/7e6542541a134c38a247346fc12bf634 2024-11-10T04:36:56,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/e08d43f8caba4eb9ab0f86e653cef1fc is 50, key is test_row_0/C:col10/1731213414164/Put/seqid=0 2024-11-10T04:36:56,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741917_1093 (size=12301) 2024-11-10T04:36:56,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/e08d43f8caba4eb9ab0f86e653cef1fc 2024-11-10T04:36:56,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/b07fdd3d329a4f8f8b6aa8093a84822b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b07fdd3d329a4f8f8b6aa8093a84822b 2024-11-10T04:36:56,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b07fdd3d329a4f8f8b6aa8093a84822b, entries=150, sequenceid=388, filesize=12.0 K 2024-11-10T04:36:56,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/7e6542541a134c38a247346fc12bf634 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/7e6542541a134c38a247346fc12bf634 2024-11-10T04:36:56,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/7e6542541a134c38a247346fc12bf634, entries=150, sequenceid=388, filesize=12.0 K 2024-11-10T04:36:56,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/e08d43f8caba4eb9ab0f86e653cef1fc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/e08d43f8caba4eb9ab0f86e653cef1fc 2024-11-10T04:36:56,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/e08d43f8caba4eb9ab0f86e653cef1fc, entries=150, sequenceid=388, filesize=12.0 K 2024-11-10T04:36:56,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 6d2efd97fd4ccb14e877b384767c0ffc in 867ms, sequenceid=388, compaction requested=false 2024-11-10T04:36:56,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:56,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T04:36:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:56,292 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T04:36:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/f5bcaa4b5a7b4bf3b73b2c6b54cf76dc is 50, key is test_row_0/A:col10/1731213415342/Put/seqid=0 2024-11-10T04:36:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T04:36:56,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741918_1094 (size=12301) 2024-11-10T04:36:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:56,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:56,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213476505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213476507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213476508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213476509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T04:36:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213476621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213476621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213476621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213476621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,706 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/f5bcaa4b5a7b4bf3b73b2c6b54cf76dc 2024-11-10T04:36:56,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/6aa03a1740a241d5a4e0829a1259a275 is 50, key is test_row_0/B:col10/1731213415342/Put/seqid=0 2024-11-10T04:36:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741919_1095 (size=12301) 2024-11-10T04:36:56,723 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/6aa03a1740a241d5a4e0829a1259a275 2024-11-10T04:36:56,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/b5cb66e305de4d45b3b495b1dafd5cb0 is 50, key is test_row_0/C:col10/1731213415342/Put/seqid=0 2024-11-10T04:36:56,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741920_1096 (size=12301) 2024-11-10T04:36:56,741 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/b5cb66e305de4d45b3b495b1dafd5cb0 2024-11-10T04:36:56,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/f5bcaa4b5a7b4bf3b73b2c6b54cf76dc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f5bcaa4b5a7b4bf3b73b2c6b54cf76dc 2024-11-10T04:36:56,754 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f5bcaa4b5a7b4bf3b73b2c6b54cf76dc, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T04:36:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-10T04:36:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T04:36:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/6aa03a1740a241d5a4e0829a1259a275 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/6aa03a1740a241d5a4e0829a1259a275 2024-11-10T04:36:56,763 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/6aa03a1740a241d5a4e0829a1259a275, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T04:36:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/b5cb66e305de4d45b3b495b1dafd5cb0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b5cb66e305de4d45b3b495b1dafd5cb0 2024-11-10T04:36:56,771 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b5cb66e305de4d45b3b495b1dafd5cb0, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T04:36:56,772 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 6d2efd97fd4ccb14e877b384767c0ffc in 480ms, sequenceid=408, compaction requested=true 2024-11-10T04:36:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:56,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-10T04:36:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-10T04:36:56,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T04:36:56,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 789 msec 2024-11-10T04:36:56,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 795 msec 2024-11-10T04:36:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:56,833 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:56,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T04:36:56,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/5e38f60b5cc94f2b9da82e3125acafa3 is 50, key is test_row_0/A:col10/1731213416832/Put/seqid=0 2024-11-10T04:36:56,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741921_1097 (size=14741) 2024-11-10T04:36:56,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/5e38f60b5cc94f2b9da82e3125acafa3 2024-11-10T04:36:56,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/cb846ac0f97f473bb4f3a46901660286 is 50, key is test_row_0/B:col10/1731213416832/Put/seqid=0 2024-11-10T04:36:56,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213476861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213476861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213476868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213476870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741922_1098 (size=12301) 2024-11-10T04:36:56,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213476981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213476981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213476981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213476981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T04:36:57,122 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:57,122 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-10T04:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T04:36:57,125 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:36:57,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:36:57,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:36:57,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213477191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213477191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213477191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213477191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T04:36:57,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T04:36:57,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:57,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:57,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:57,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:57,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:36:57,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/cb846ac0f97f473bb4f3a46901660286 2024-11-10T04:36:57,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/a9cf5e2ef08f46a7a50d14aee2f886bb is 50, key is test_row_0/C:col10/1731213416832/Put/seqid=0 2024-11-10T04:36:57,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741923_1099 (size=12301) 2024-11-10T04:36:57,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/a9cf5e2ef08f46a7a50d14aee2f886bb 2024-11-10T04:36:57,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/5e38f60b5cc94f2b9da82e3125acafa3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5e38f60b5cc94f2b9da82e3125acafa3 2024-11-10T04:36:57,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5e38f60b5cc94f2b9da82e3125acafa3, entries=200, sequenceid=429, filesize=14.4 K 2024-11-10T04:36:57,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/cb846ac0f97f473bb4f3a46901660286 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cb846ac0f97f473bb4f3a46901660286 2024-11-10T04:36:57,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cb846ac0f97f473bb4f3a46901660286, entries=150, sequenceid=429, filesize=12.0 K 2024-11-10T04:36:57,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/a9cf5e2ef08f46a7a50d14aee2f886bb as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a9cf5e2ef08f46a7a50d14aee2f886bb 2024-11-10T04:36:57,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a9cf5e2ef08f46a7a50d14aee2f886bb, entries=150, sequenceid=429, filesize=12.0 K 2024-11-10T04:36:57,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6d2efd97fd4ccb14e877b384767c0ffc in 494ms, sequenceid=429, compaction requested=true 2024-11-10T04:36:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:36:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:57,327 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:36:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:36:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:36:57,327 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:36:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:57,329 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:36:57,329 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/A is initiating minor compaction (all files) 2024-11-10T04:36:57,329 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/A in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:57,329 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/213372a5cf1a4484b5f68db6e4f4d98a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b07fdd3d329a4f8f8b6aa8093a84822b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f5bcaa4b5a7b4bf3b73b2c6b54cf76dc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5e38f60b5cc94f2b9da82e3125acafa3] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=51.3 K 2024-11-10T04:36:57,329 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:36:57,329 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/B is initiating minor compaction (all files) 2024-11-10T04:36:57,329 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/B in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:57,329 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 213372a5cf1a4484b5f68db6e4f4d98a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731213413001 2024-11-10T04:36:57,329 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a45b3fefe38b44b086453a66198090f6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/7e6542541a134c38a247346fc12bf634, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/6aa03a1740a241d5a4e0829a1259a275, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cb846ac0f97f473bb4f3a46901660286] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=48.9 K 2024-11-10T04:36:57,330 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a45b3fefe38b44b086453a66198090f6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731213413001 2024-11-10T04:36:57,330 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b07fdd3d329a4f8f8b6aa8093a84822b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731213414164 2024-11-10T04:36:57,330 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e6542541a134c38a247346fc12bf634, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731213414164 2024-11-10T04:36:57,331 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5bcaa4b5a7b4bf3b73b2c6b54cf76dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731213415328 2024-11-10T04:36:57,331 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aa03a1740a241d5a4e0829a1259a275, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731213415328 2024-11-10T04:36:57,332 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cb846ac0f97f473bb4f3a46901660286, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731213416498 2024-11-10T04:36:57,332 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e38f60b5cc94f2b9da82e3125acafa3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731213416498 2024-11-10T04:36:57,347 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:57,347 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:57,348 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/151bee749404440e926b144f7cabc02c is 50, key is test_row_0/B:col10/1731213416832/Put/seqid=0 2024-11-10T04:36:57,348 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8090bc135b2c4bf38dbc70a98d08d5c6 is 50, key is test_row_0/A:col10/1731213416832/Put/seqid=0 2024-11-10T04:36:57,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741924_1100 (size=13289) 2024-11-10T04:36:57,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741925_1101 (size=13289) 2024-11-10T04:36:57,363 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/151bee749404440e926b144f7cabc02c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/151bee749404440e926b144f7cabc02c 2024-11-10T04:36:57,369 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/B of 6d2efd97fd4ccb14e877b384767c0ffc into 151bee749404440e926b144f7cabc02c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:57,369 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:57,369 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/B, priority=12, startTime=1731213417327; duration=0sec 2024-11-10T04:36:57,370 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:36:57,370 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:B 2024-11-10T04:36:57,370 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:36:57,371 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:36:57,371 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/C is initiating minor compaction (all files) 2024-11-10T04:36:57,372 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/C in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:57,372 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ca2b073828f49d6a2e676c03efb7a6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/e08d43f8caba4eb9ab0f86e653cef1fc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b5cb66e305de4d45b3b495b1dafd5cb0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a9cf5e2ef08f46a7a50d14aee2f886bb] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=48.9 K 2024-11-10T04:36:57,372 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca2b073828f49d6a2e676c03efb7a6f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731213413001 2024-11-10T04:36:57,373 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e08d43f8caba4eb9ab0f86e653cef1fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731213414164 2024-11-10T04:36:57,373 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b5cb66e305de4d45b3b495b1dafd5cb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731213415328 2024-11-10T04:36:57,374 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a9cf5e2ef08f46a7a50d14aee2f886bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731213416498 2024-11-10T04:36:57,382 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#C#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:36:57,383 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/1ef1528a1bfb418bb9c2bea80677eba2 is 50, key is test_row_0/C:col10/1731213416832/Put/seqid=0 2024-11-10T04:36:57,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741926_1102 (size=13289) 2024-11-10T04:36:57,396 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/1ef1528a1bfb418bb9c2bea80677eba2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1ef1528a1bfb418bb9c2bea80677eba2 2024-11-10T04:36:57,403 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/C of 6d2efd97fd4ccb14e877b384767c0ffc into 1ef1528a1bfb418bb9c2bea80677eba2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:57,403 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:57,403 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/C, priority=12, startTime=1731213417327; duration=0sec 2024-11-10T04:36:57,403 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:57,404 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:36:57,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T04:36:57,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:57,433 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T04:36:57,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/c382f797b379445f8f27cd65470b25cd is 50, key is test_row_0/A:col10/1731213416856/Put/seqid=0 2024-11-10T04:36:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T04:36:57,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741927_1103 (size=12301) 2024-11-10T04:36:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:57,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. as already flushing 2024-11-10T04:36:57,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213477525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213477526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213477527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213477528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,541 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-10T04:36:57,541 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:36:57,542 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-10T04:36:57,542 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:36:57,545 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-10T04:36:57,545 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:36:57,546 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-10T04:36:57,546 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:36:57,547 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:57,547 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:57,547 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:57,547 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:57,548 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:57,548 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:57,548 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:57,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:36:57,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:36:57,548 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:57,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:36:57,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:36:57,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213477641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213477641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213477641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213477641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T04:36:57,761 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8090bc135b2c4bf38dbc70a98d08d5c6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8090bc135b2c4bf38dbc70a98d08d5c6 2024-11-10T04:36:57,766 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/A of 6d2efd97fd4ccb14e877b384767c0ffc into 8090bc135b2c4bf38dbc70a98d08d5c6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:36:57,766 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:57,766 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/A, priority=12, startTime=1731213417327; duration=0sec 2024-11-10T04:36:57,766 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:36:57,766 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:A 2024-11-10T04:36:57,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213477851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213477851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213477851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213477851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:57,853 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/c382f797b379445f8f27cd65470b25cd 2024-11-10T04:36:57,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/ae3008927be94621ad31bd2dc158a518 is 50, key is test_row_0/B:col10/1731213416856/Put/seqid=0 2024-11-10T04:36:57,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741928_1104 (size=12301) 2024-11-10T04:36:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56126 deadline: 1731213478101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:58,101 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 6d2efd97fd4ccb14e877b384767c0ffc of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18322 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:36:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:58,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213478161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:58,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:58,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213478161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213478161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:58,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213478161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T04:36:58,264 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/ae3008927be94621ad31bd2dc158a518 2024-11-10T04:36:58,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/5ab3a59d10bf4672bd053f0f72208f4d is 50, key is test_row_0/C:col10/1731213416856/Put/seqid=0 2024-11-10T04:36:58,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741929_1105 (size=12301) 2024-11-10T04:36:58,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56158 deadline: 1731213478671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:58,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:58,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56138 deadline: 1731213478671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731213478671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:58,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:36:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56156 deadline: 1731213478671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:36:58,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:36:58,677 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/5ab3a59d10bf4672bd053f0f72208f4d 2024-11-10T04:36:58,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/c382f797b379445f8f27cd65470b25cd as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c382f797b379445f8f27cd65470b25cd 2024-11-10T04:36:58,689 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c382f797b379445f8f27cd65470b25cd, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T04:36:58,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/ae3008927be94621ad31bd2dc158a518 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ae3008927be94621ad31bd2dc158a518 2024-11-10T04:36:58,695 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ae3008927be94621ad31bd2dc158a518, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T04:36:58,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/5ab3a59d10bf4672bd053f0f72208f4d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ab3a59d10bf4672bd053f0f72208f4d 2024-11-10T04:36:58,700 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ab3a59d10bf4672bd053f0f72208f4d, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T04:36:58,701 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6d2efd97fd4ccb14e877b384767c0ffc in 1269ms, sequenceid=446, compaction requested=false 2024-11-10T04:36:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:36:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:36:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-10T04:36:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-10T04:36:58,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-10T04:36:58,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5770 sec 2024-11-10T04:36:58,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.5830 sec 2024-11-10T04:36:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T04:36:59,271 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:36:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:36:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:36:59,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T04:36:59,683 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-10T04:36:59,683 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-10T04:36:59,683 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-10T04:36:59,683 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:36:59,683 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:36:59,683 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:36:59,683 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:59,683 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:59,683 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:59,683 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:59,683 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:59,683 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:59,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:36:59,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:36:59,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:36:59,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/a4dfff02980e4419ac36a6b18bcf9bf4 is 50, key is test_row_0/A:col10/1731213417527/Put/seqid=0 2024-11-10T04:36:59,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741930_1106 (size=12301) 2024-11-10T04:36:59,693 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-10T04:36:59,693 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:36:59,693 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:59,693 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:36:59,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:00,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/a4dfff02980e4419ac36a6b18bcf9bf4 2024-11-10T04:37:00,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/25a46af1036e4925b5323e3b5b974028 is 50, key is test_row_0/B:col10/1731213417527/Put/seqid=0 2024-11-10T04:37:00,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741931_1107 (size=12301) 2024-11-10T04:37:00,305 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:37:00,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/25a46af1036e4925b5323e3b5b974028 2024-11-10T04:37:00,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/a37134c9de294561b10d0af09bc1f188 is 50, key is test_row_0/C:col10/1731213417527/Put/seqid=0 2024-11-10T04:37:00,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741932_1108 (size=12301) 2024-11-10T04:37:00,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/a37134c9de294561b10d0af09bc1f188 2024-11-10T04:37:00,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/a4dfff02980e4419ac36a6b18bcf9bf4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a4dfff02980e4419ac36a6b18bcf9bf4 2024-11-10T04:37:00,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a4dfff02980e4419ac36a6b18bcf9bf4, entries=150, sequenceid=471, filesize=12.0 K 2024-11-10T04:37:00,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/25a46af1036e4925b5323e3b5b974028 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/25a46af1036e4925b5323e3b5b974028 2024-11-10T04:37:00,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/25a46af1036e4925b5323e3b5b974028, entries=150, sequenceid=471, filesize=12.0 K 2024-11-10T04:37:00,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/a37134c9de294561b10d0af09bc1f188 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a37134c9de294561b10d0af09bc1f188 2024-11-10T04:37:00,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a37134c9de294561b10d0af09bc1f188, entries=150, sequenceid=471, filesize=12.0 K 2024-11-10T04:37:00,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for 6d2efd97fd4ccb14e877b384767c0ffc in 1259ms, sequenceid=471, compaction requested=true 2024-11-10T04:37:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:37:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:00,942 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:00,942 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d2efd97fd4ccb14e877b384767c0ffc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:00,943 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:00,943 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:00,943 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/B is initiating minor compaction (all files) 2024-11-10T04:37:00,943 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/A is initiating minor compaction (all files) 2024-11-10T04:37:00,943 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/B in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:37:00,943 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/A in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:37:00,943 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/151bee749404440e926b144f7cabc02c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ae3008927be94621ad31bd2dc158a518, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/25a46af1036e4925b5323e3b5b974028] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=37.0 K 2024-11-10T04:37:00,943 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8090bc135b2c4bf38dbc70a98d08d5c6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c382f797b379445f8f27cd65470b25cd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a4dfff02980e4419ac36a6b18bcf9bf4] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=37.0 K 2024-11-10T04:37:00,944 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 151bee749404440e926b144f7cabc02c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731213416498 2024-11-10T04:37:00,944 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8090bc135b2c4bf38dbc70a98d08d5c6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731213416498 2024-11-10T04:37:00,944 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ae3008927be94621ad31bd2dc158a518, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731213416856 2024-11-10T04:37:00,944 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c382f797b379445f8f27cd65470b25cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731213416856 2024-11-10T04:37:00,945 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 25a46af1036e4925b5323e3b5b974028, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731213417527 2024-11-10T04:37:00,945 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4dfff02980e4419ac36a6b18bcf9bf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731213417527 2024-11-10T04:37:00,954 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:00,955 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#A#compaction#97 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:00,955 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/cd6959bb2757429f8800d8d5b565c79e is 50, key is test_row_0/B:col10/1731213417527/Put/seqid=0 2024-11-10T04:37:00,955 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8a7291b84967434690bab122cff6d281 is 50, key is test_row_0/A:col10/1731213417527/Put/seqid=0 2024-11-10T04:37:00,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741934_1110 (size=13391) 2024-11-10T04:37:00,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741933_1109 (size=13391) 2024-11-10T04:37:00,968 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/8a7291b84967434690bab122cff6d281 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8a7291b84967434690bab122cff6d281 2024-11-10T04:37:00,973 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/A of 6d2efd97fd4ccb14e877b384767c0ffc into 8a7291b84967434690bab122cff6d281(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:00,973 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:37:00,973 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/A, priority=13, startTime=1731213420942; duration=0sec 2024-11-10T04:37:00,974 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:00,974 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:A 2024-11-10T04:37:00,974 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:00,975 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:00,975 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d2efd97fd4ccb14e877b384767c0ffc/C is initiating minor compaction (all files) 2024-11-10T04:37:00,975 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d2efd97fd4ccb14e877b384767c0ffc/C in TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:37:00,975 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1ef1528a1bfb418bb9c2bea80677eba2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ab3a59d10bf4672bd053f0f72208f4d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a37134c9de294561b10d0af09bc1f188] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp, totalSize=37.0 K 2024-11-10T04:37:00,975 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ef1528a1bfb418bb9c2bea80677eba2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731213416498 2024-11-10T04:37:00,976 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ab3a59d10bf4672bd053f0f72208f4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731213416856 2024-11-10T04:37:00,976 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a37134c9de294561b10d0af09bc1f188, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731213417527 2024-11-10T04:37:00,983 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d2efd97fd4ccb14e877b384767c0ffc#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:00,983 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/bfb032111a10461baefea5c0d46886f6 is 50, key is test_row_0/C:col10/1731213417527/Put/seqid=0 2024-11-10T04:37:00,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741935_1111 (size=13391) 2024-11-10T04:37:01,368 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/cd6959bb2757429f8800d8d5b565c79e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cd6959bb2757429f8800d8d5b565c79e 2024-11-10T04:37:01,373 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/B of 6d2efd97fd4ccb14e877b384767c0ffc into cd6959bb2757429f8800d8d5b565c79e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:01,373 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:37:01,373 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/B, priority=13, startTime=1731213420942; duration=0sec 2024-11-10T04:37:01,373 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:01,373 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:B 2024-11-10T04:37:01,392 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/bfb032111a10461baefea5c0d46886f6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/bfb032111a10461baefea5c0d46886f6 2024-11-10T04:37:01,397 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d2efd97fd4ccb14e877b384767c0ffc/C of 6d2efd97fd4ccb14e877b384767c0ffc into bfb032111a10461baefea5c0d46886f6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:01,397 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d2efd97fd4ccb14e877b384767c0ffc: 2024-11-10T04:37:01,397 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., storeName=6d2efd97fd4ccb14e877b384767c0ffc/C, priority=13, startTime=1731213420942; duration=0sec 2024-11-10T04:37:01,397 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:01,397 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d2efd97fd4ccb14e877b384767c0ffc:C 2024-11-10T04:37:08,113 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-10T04:37:08,113 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:08,113 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:08,114 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:08,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3537 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3462 2024-11-10T04:37:08,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T04:37:08,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1519 2024-11-10T04:37:08,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4551 rows 2024-11-10T04:37:08,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1514 2024-11-10T04:37:08,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4538 rows 2024-11-10T04:37:08,115 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:37:08,115 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:37:08,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:08,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:08,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:08,120 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T04:37:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:08,129 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213428129"}]},"ts":"1731213428129"} 2024-11-10T04:37:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T04:37:08,131 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T04:37:08,131 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T04:37:08,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T04:37:08,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d2efd97fd4ccb14e877b384767c0ffc, UNASSIGN}] 2024-11-10T04:37:08,138 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d2efd97fd4ccb14e877b384767c0ffc, UNASSIGN 2024-11-10T04:37:08,139 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6d2efd97fd4ccb14e877b384767c0ffc, regionState=CLOSING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:08,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d2efd97fd4ccb14e877b384767c0ffc, UNASSIGN because future has completed 2024-11-10T04:37:08,141 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T04:37:08,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:37:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T04:37:08,298 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:37:08,298 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T04:37:08,299 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 6d2efd97fd4ccb14e877b384767c0ffc, disabling compactions & flushes 2024-11-10T04:37:08,299 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:37:08,299 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:37:08,299 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. after waiting 0 ms 2024-11-10T04:37:08,299 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:37:08,299 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 6d2efd97fd4ccb14e877b384767c0ffc 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-10T04:37:08,303 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/3cb5a4ac6c674672a34b94fb59287c79 is 50, key is test_row_1/A:col10/1731213428112/Put/seqid=0 2024-11-10T04:37:08,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741936_1112 (size=9857) 2024-11-10T04:37:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T04:37:08,708 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/3cb5a4ac6c674672a34b94fb59287c79 2024-11-10T04:37:08,716 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/e3d1aab2d98148e4ba186dfcdb532b55 is 50, key is test_row_1/B:col10/1731213428112/Put/seqid=0 2024-11-10T04:37:08,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741937_1113 (size=9857) 2024-11-10T04:37:08,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T04:37:09,121 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/e3d1aab2d98148e4ba186dfcdb532b55 2024-11-10T04:37:09,130 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/665f3ccfdd08453195881fbbaa4e9b91 is 50, key is test_row_1/C:col10/1731213428112/Put/seqid=0 2024-11-10T04:37:09,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741938_1114 (size=9857) 2024-11-10T04:37:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T04:37:09,534 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/665f3ccfdd08453195881fbbaa4e9b91 2024-11-10T04:37:09,539 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/A/3cb5a4ac6c674672a34b94fb59287c79 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/3cb5a4ac6c674672a34b94fb59287c79 2024-11-10T04:37:09,543 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/3cb5a4ac6c674672a34b94fb59287c79, entries=100, sequenceid=479, filesize=9.6 K 2024-11-10T04:37:09,544 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/B/e3d1aab2d98148e4ba186dfcdb532b55 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/e3d1aab2d98148e4ba186dfcdb532b55 2024-11-10T04:37:09,548 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/e3d1aab2d98148e4ba186dfcdb532b55, entries=100, sequenceid=479, filesize=9.6 K 2024-11-10T04:37:09,549 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/.tmp/C/665f3ccfdd08453195881fbbaa4e9b91 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/665f3ccfdd08453195881fbbaa4e9b91 2024-11-10T04:37:09,553 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/665f3ccfdd08453195881fbbaa4e9b91, entries=100, sequenceid=479, filesize=9.6 K 2024-11-10T04:37:09,554 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 6d2efd97fd4ccb14e877b384767c0ffc in 1255ms, sequenceid=479, compaction requested=false 2024-11-10T04:37:09,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/af089e3731ee4e17858e433fda8f27b6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/81c1595d72084ed8b97599c267fd8443, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/d2639fee20464c5a858c822eb038d484, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/db471b6d187c43559ee33c8c23452db5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8e0dbb5971394ba48eba19d1c271f40e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/cb3bb56a23014f238079ba66f22baac1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a78a019721204882a2eac225fe6b5eb2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6655f07490f7479abde69388c75769d0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c5ad68ea14c94f4e822a7d3c86b8e5c3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5efb4edc2675426ea21add4c521a9265, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6f8368ec87444933a823d5f4ab00bf75, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b787537def8a4384991a0f59e1bcde96, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/28e09bd3bdaf4de4ae875d2f2246e200, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b21d19292b3c437590d2723f37b60565, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/05f278b677d146f3af37ad087d0b05d1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/e8c939fc17eb42f396fdf9e9c23ac42e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f7c8b0653653411c82412200c51a9fc5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57a76a743c0c4968b9b4e513cd898a59, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8c5c6a4440df4b0d83f4cb4266f5abe9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/4adc06991e784982a95e157857f5da7d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57090a8fef944f5eacaba22cb1265dfe, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6820a58986094f27a6e31a8b80a210bc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/eed3ca8a10264a14aa29ebb72b2332e7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/419eebbf83d1409eb6b6a5621d098925, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/213372a5cf1a4484b5f68db6e4f4d98a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/432fdf1592274ad5b5de671f68f33142, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b07fdd3d329a4f8f8b6aa8093a84822b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f5bcaa4b5a7b4bf3b73b2c6b54cf76dc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5e38f60b5cc94f2b9da82e3125acafa3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8090bc135b2c4bf38dbc70a98d08d5c6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c382f797b379445f8f27cd65470b25cd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a4dfff02980e4419ac36a6b18bcf9bf4] to archive 2024-11-10T04:37:09,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:37:09,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/af089e3731ee4e17858e433fda8f27b6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/af089e3731ee4e17858e433fda8f27b6 2024-11-10T04:37:09,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/81c1595d72084ed8b97599c267fd8443 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/81c1595d72084ed8b97599c267fd8443 2024-11-10T04:37:09,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/d2639fee20464c5a858c822eb038d484 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/d2639fee20464c5a858c822eb038d484 2024-11-10T04:37:09,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/db471b6d187c43559ee33c8c23452db5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/db471b6d187c43559ee33c8c23452db5 2024-11-10T04:37:09,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8e0dbb5971394ba48eba19d1c271f40e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8e0dbb5971394ba48eba19d1c271f40e 2024-11-10T04:37:09,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/cb3bb56a23014f238079ba66f22baac1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/cb3bb56a23014f238079ba66f22baac1 2024-11-10T04:37:09,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a78a019721204882a2eac225fe6b5eb2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a78a019721204882a2eac225fe6b5eb2 2024-11-10T04:37:09,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6655f07490f7479abde69388c75769d0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6655f07490f7479abde69388c75769d0 2024-11-10T04:37:09,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c5ad68ea14c94f4e822a7d3c86b8e5c3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c5ad68ea14c94f4e822a7d3c86b8e5c3 2024-11-10T04:37:09,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5efb4edc2675426ea21add4c521a9265 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5efb4edc2675426ea21add4c521a9265 2024-11-10T04:37:09,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6f8368ec87444933a823d5f4ab00bf75 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6f8368ec87444933a823d5f4ab00bf75 2024-11-10T04:37:09,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b787537def8a4384991a0f59e1bcde96 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b787537def8a4384991a0f59e1bcde96 2024-11-10T04:37:09,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/28e09bd3bdaf4de4ae875d2f2246e200 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/28e09bd3bdaf4de4ae875d2f2246e200 2024-11-10T04:37:09,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b21d19292b3c437590d2723f37b60565 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b21d19292b3c437590d2723f37b60565 2024-11-10T04:37:09,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/05f278b677d146f3af37ad087d0b05d1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/05f278b677d146f3af37ad087d0b05d1 2024-11-10T04:37:09,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/e8c939fc17eb42f396fdf9e9c23ac42e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/e8c939fc17eb42f396fdf9e9c23ac42e 2024-11-10T04:37:09,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f7c8b0653653411c82412200c51a9fc5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f7c8b0653653411c82412200c51a9fc5 2024-11-10T04:37:09,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57a76a743c0c4968b9b4e513cd898a59 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57a76a743c0c4968b9b4e513cd898a59 2024-11-10T04:37:09,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8c5c6a4440df4b0d83f4cb4266f5abe9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8c5c6a4440df4b0d83f4cb4266f5abe9 2024-11-10T04:37:09,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/4adc06991e784982a95e157857f5da7d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/4adc06991e784982a95e157857f5da7d 2024-11-10T04:37:09,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57090a8fef944f5eacaba22cb1265dfe to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/57090a8fef944f5eacaba22cb1265dfe 2024-11-10T04:37:09,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6820a58986094f27a6e31a8b80a210bc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/6820a58986094f27a6e31a8b80a210bc 2024-11-10T04:37:09,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/eed3ca8a10264a14aa29ebb72b2332e7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/eed3ca8a10264a14aa29ebb72b2332e7 2024-11-10T04:37:09,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/419eebbf83d1409eb6b6a5621d098925 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/419eebbf83d1409eb6b6a5621d098925 2024-11-10T04:37:09,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/213372a5cf1a4484b5f68db6e4f4d98a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/213372a5cf1a4484b5f68db6e4f4d98a 2024-11-10T04:37:09,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/432fdf1592274ad5b5de671f68f33142 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/432fdf1592274ad5b5de671f68f33142 2024-11-10T04:37:09,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b07fdd3d329a4f8f8b6aa8093a84822b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/b07fdd3d329a4f8f8b6aa8093a84822b 2024-11-10T04:37:09,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f5bcaa4b5a7b4bf3b73b2c6b54cf76dc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/f5bcaa4b5a7b4bf3b73b2c6b54cf76dc 2024-11-10T04:37:09,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5e38f60b5cc94f2b9da82e3125acafa3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/5e38f60b5cc94f2b9da82e3125acafa3 2024-11-10T04:37:09,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8090bc135b2c4bf38dbc70a98d08d5c6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8090bc135b2c4bf38dbc70a98d08d5c6 2024-11-10T04:37:09,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c382f797b379445f8f27cd65470b25cd to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/c382f797b379445f8f27cd65470b25cd 2024-11-10T04:37:09,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a4dfff02980e4419ac36a6b18bcf9bf4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/a4dfff02980e4419ac36a6b18bcf9bf4 2024-11-10T04:37:09,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/476370440ad445d1bc77d6585eeea561, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/254e14a3c704487e862cda7bcc8b9893, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/759866fcd2a1429fab0068fec47ed337, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/89a5c21548234bfeb6cb62357ac4b4a4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/63273bd9878f44c182ce6a3fbbd6700a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ccdc449167ae460e9e802889bfbe762d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/0e2742f6ec3d4bdc8638b2ecce918fff, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c7ae9ac5b2be4b7f985b5abb9905c0e0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b66aaac446bf43edbfd353ea89a2be38, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5f07af19421e4976af81c7ee83a825d2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5685bbdb433e478c96bd04f9058c0849, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9c0f896a61ef43e4b993f9fcaa3c25bd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/edaaf027ebe8476db81d73683dfcd036, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cf09cdd9038e4c89b03628e911f3929d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/581caffd42c84f7aa21f79c30aa26e30, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/bcafc56cd95b47178a3a487e0b055736, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/1bdf30c4e3ec4d848a621bddb9950ad6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/dbb86552188c4ec5aedb6f226e543b42, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/142e6dd96fe945d68aafe39988f53325, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c23c4d4860df4bb7acf4b5316db40a09, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b8f2bb7d4cfd4d38b5dc3c6adb24a5da, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/e0367834a93244ca85683ac3484efac2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a88e6f9f519a45a89a5ea50103e935cb, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9a22c2773ae74266a13ee66bbe973ee2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a45b3fefe38b44b086453a66198090f6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/91992f76c8fe4d9ba1054d983431af8c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/7e6542541a134c38a247346fc12bf634, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/6aa03a1740a241d5a4e0829a1259a275, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/151bee749404440e926b144f7cabc02c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cb846ac0f97f473bb4f3a46901660286, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ae3008927be94621ad31bd2dc158a518, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/25a46af1036e4925b5323e3b5b974028] to archive 2024-11-10T04:37:09,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:37:09,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/476370440ad445d1bc77d6585eeea561 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/476370440ad445d1bc77d6585eeea561 2024-11-10T04:37:09,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/254e14a3c704487e862cda7bcc8b9893 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/254e14a3c704487e862cda7bcc8b9893 2024-11-10T04:37:09,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/759866fcd2a1429fab0068fec47ed337 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/759866fcd2a1429fab0068fec47ed337 2024-11-10T04:37:09,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/89a5c21548234bfeb6cb62357ac4b4a4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/89a5c21548234bfeb6cb62357ac4b4a4 2024-11-10T04:37:09,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/63273bd9878f44c182ce6a3fbbd6700a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/63273bd9878f44c182ce6a3fbbd6700a 2024-11-10T04:37:09,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ccdc449167ae460e9e802889bfbe762d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ccdc449167ae460e9e802889bfbe762d 2024-11-10T04:37:09,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/0e2742f6ec3d4bdc8638b2ecce918fff to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/0e2742f6ec3d4bdc8638b2ecce918fff 2024-11-10T04:37:09,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c7ae9ac5b2be4b7f985b5abb9905c0e0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c7ae9ac5b2be4b7f985b5abb9905c0e0 2024-11-10T04:37:09,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b66aaac446bf43edbfd353ea89a2be38 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b66aaac446bf43edbfd353ea89a2be38 2024-11-10T04:37:09,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5f07af19421e4976af81c7ee83a825d2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5f07af19421e4976af81c7ee83a825d2 2024-11-10T04:37:09,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5685bbdb433e478c96bd04f9058c0849 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/5685bbdb433e478c96bd04f9058c0849 2024-11-10T04:37:09,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9c0f896a61ef43e4b993f9fcaa3c25bd to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9c0f896a61ef43e4b993f9fcaa3c25bd 2024-11-10T04:37:09,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/edaaf027ebe8476db81d73683dfcd036 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/edaaf027ebe8476db81d73683dfcd036 2024-11-10T04:37:09,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cf09cdd9038e4c89b03628e911f3929d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cf09cdd9038e4c89b03628e911f3929d 2024-11-10T04:37:09,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/581caffd42c84f7aa21f79c30aa26e30 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/581caffd42c84f7aa21f79c30aa26e30 2024-11-10T04:37:09,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/bcafc56cd95b47178a3a487e0b055736 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/bcafc56cd95b47178a3a487e0b055736 2024-11-10T04:37:09,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/1bdf30c4e3ec4d848a621bddb9950ad6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/1bdf30c4e3ec4d848a621bddb9950ad6 2024-11-10T04:37:09,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/dbb86552188c4ec5aedb6f226e543b42 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/dbb86552188c4ec5aedb6f226e543b42 2024-11-10T04:37:09,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/142e6dd96fe945d68aafe39988f53325 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/142e6dd96fe945d68aafe39988f53325 2024-11-10T04:37:09,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c23c4d4860df4bb7acf4b5316db40a09 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/c23c4d4860df4bb7acf4b5316db40a09 2024-11-10T04:37:09,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b8f2bb7d4cfd4d38b5dc3c6adb24a5da to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/b8f2bb7d4cfd4d38b5dc3c6adb24a5da 2024-11-10T04:37:09,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/e0367834a93244ca85683ac3484efac2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/e0367834a93244ca85683ac3484efac2 2024-11-10T04:37:09,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a88e6f9f519a45a89a5ea50103e935cb to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a88e6f9f519a45a89a5ea50103e935cb 2024-11-10T04:37:09,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9a22c2773ae74266a13ee66bbe973ee2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/9a22c2773ae74266a13ee66bbe973ee2 2024-11-10T04:37:09,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a45b3fefe38b44b086453a66198090f6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/a45b3fefe38b44b086453a66198090f6 2024-11-10T04:37:09,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/91992f76c8fe4d9ba1054d983431af8c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/91992f76c8fe4d9ba1054d983431af8c 2024-11-10T04:37:09,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/7e6542541a134c38a247346fc12bf634 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/7e6542541a134c38a247346fc12bf634 2024-11-10T04:37:09,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/6aa03a1740a241d5a4e0829a1259a275 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/6aa03a1740a241d5a4e0829a1259a275 2024-11-10T04:37:09,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/151bee749404440e926b144f7cabc02c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/151bee749404440e926b144f7cabc02c 2024-11-10T04:37:09,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cb846ac0f97f473bb4f3a46901660286 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cb846ac0f97f473bb4f3a46901660286 2024-11-10T04:37:09,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ae3008927be94621ad31bd2dc158a518 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/ae3008927be94621ad31bd2dc158a518 2024-11-10T04:37:09,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/25a46af1036e4925b5323e3b5b974028 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/25a46af1036e4925b5323e3b5b974028 2024-11-10T04:37:09,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/420ec416a3be4f35887696da839de68b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dd31c72a7676467795a75d5c8eaf68af, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/6f43e1a162af408987dc7f2de2afb1ec, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/40f1b730db9d4a3d8d4d7755a452b0f7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/faac07d70f76455992ba93ef0a1e2373, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dc6a418717ee4d3585ba6788b4b955f2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/51a89506e538477ab8e80b4c94b78bfb, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/9c546730c2684f7ba157025bc52cfcab, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/ae926e56c92f46b48f3174ab9ae7257b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/bf6ae0199c7d4b588404f216aea4d92d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b18f691db64f413c908244281192aa5b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/8f49c9bf2df542428b85048b0ae0e985, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/4f83e7512d3e4d3abff909572fb30063, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0aff9481ad614b66a75311f249af2512, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/66d978064dca4552bddefadf1e2570d4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/adde20b42a274d5293a41963b836bb11, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/69bbd2cb3ecf4447986a05a19e8a1f6e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1d18535a037d48349d7a224669d66b0e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/142ba19a55f545518f0b58ed72b2a661, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a585506c2d514d5d86cf5071bca0b8e4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/210270c117bd46ad90710a3b376cb241, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/f595c7d6f54040bfbd45cd1795c71164, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/019f901b37db493e87864d7e2b7762d4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0e2c588a3c2a4442a1c357b1a5decc8f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ca2b073828f49d6a2e676c03efb7a6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/439e96cbc2664700b5c55ee22e6066ca, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/e08d43f8caba4eb9ab0f86e653cef1fc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b5cb66e305de4d45b3b495b1dafd5cb0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1ef1528a1bfb418bb9c2bea80677eba2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a9cf5e2ef08f46a7a50d14aee2f886bb, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ab3a59d10bf4672bd053f0f72208f4d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a37134c9de294561b10d0af09bc1f188] to archive 2024-11-10T04:37:09,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:37:09,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/420ec416a3be4f35887696da839de68b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/420ec416a3be4f35887696da839de68b 2024-11-10T04:37:09,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dd31c72a7676467795a75d5c8eaf68af to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dd31c72a7676467795a75d5c8eaf68af 2024-11-10T04:37:09,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/6f43e1a162af408987dc7f2de2afb1ec to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/6f43e1a162af408987dc7f2de2afb1ec 2024-11-10T04:37:09,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/40f1b730db9d4a3d8d4d7755a452b0f7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/40f1b730db9d4a3d8d4d7755a452b0f7 2024-11-10T04:37:09,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/faac07d70f76455992ba93ef0a1e2373 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/faac07d70f76455992ba93ef0a1e2373 2024-11-10T04:37:09,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dc6a418717ee4d3585ba6788b4b955f2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/dc6a418717ee4d3585ba6788b4b955f2 2024-11-10T04:37:09,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/51a89506e538477ab8e80b4c94b78bfb to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/51a89506e538477ab8e80b4c94b78bfb 2024-11-10T04:37:09,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/9c546730c2684f7ba157025bc52cfcab to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/9c546730c2684f7ba157025bc52cfcab 2024-11-10T04:37:09,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/ae926e56c92f46b48f3174ab9ae7257b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/ae926e56c92f46b48f3174ab9ae7257b 2024-11-10T04:37:09,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/bf6ae0199c7d4b588404f216aea4d92d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/bf6ae0199c7d4b588404f216aea4d92d 2024-11-10T04:37:09,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b18f691db64f413c908244281192aa5b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b18f691db64f413c908244281192aa5b 2024-11-10T04:37:09,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/8f49c9bf2df542428b85048b0ae0e985 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/8f49c9bf2df542428b85048b0ae0e985 2024-11-10T04:37:09,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/4f83e7512d3e4d3abff909572fb30063 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/4f83e7512d3e4d3abff909572fb30063 2024-11-10T04:37:09,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0aff9481ad614b66a75311f249af2512 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0aff9481ad614b66a75311f249af2512 2024-11-10T04:37:09,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/66d978064dca4552bddefadf1e2570d4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/66d978064dca4552bddefadf1e2570d4 2024-11-10T04:37:09,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/adde20b42a274d5293a41963b836bb11 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/adde20b42a274d5293a41963b836bb11 2024-11-10T04:37:09,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/69bbd2cb3ecf4447986a05a19e8a1f6e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/69bbd2cb3ecf4447986a05a19e8a1f6e 2024-11-10T04:37:09,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1d18535a037d48349d7a224669d66b0e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1d18535a037d48349d7a224669d66b0e 2024-11-10T04:37:09,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/142ba19a55f545518f0b58ed72b2a661 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/142ba19a55f545518f0b58ed72b2a661 2024-11-10T04:37:09,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a585506c2d514d5d86cf5071bca0b8e4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a585506c2d514d5d86cf5071bca0b8e4 2024-11-10T04:37:09,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/210270c117bd46ad90710a3b376cb241 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/210270c117bd46ad90710a3b376cb241 2024-11-10T04:37:09,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/f595c7d6f54040bfbd45cd1795c71164 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/f595c7d6f54040bfbd45cd1795c71164 2024-11-10T04:37:09,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/019f901b37db493e87864d7e2b7762d4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/019f901b37db493e87864d7e2b7762d4 2024-11-10T04:37:09,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0e2c588a3c2a4442a1c357b1a5decc8f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/0e2c588a3c2a4442a1c357b1a5decc8f 2024-11-10T04:37:09,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ca2b073828f49d6a2e676c03efb7a6f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ca2b073828f49d6a2e676c03efb7a6f 2024-11-10T04:37:09,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/439e96cbc2664700b5c55ee22e6066ca to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/439e96cbc2664700b5c55ee22e6066ca 2024-11-10T04:37:09,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/e08d43f8caba4eb9ab0f86e653cef1fc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/e08d43f8caba4eb9ab0f86e653cef1fc 2024-11-10T04:37:09,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b5cb66e305de4d45b3b495b1dafd5cb0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/b5cb66e305de4d45b3b495b1dafd5cb0 2024-11-10T04:37:09,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1ef1528a1bfb418bb9c2bea80677eba2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/1ef1528a1bfb418bb9c2bea80677eba2 2024-11-10T04:37:09,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a9cf5e2ef08f46a7a50d14aee2f886bb to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a9cf5e2ef08f46a7a50d14aee2f886bb 2024-11-10T04:37:09,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ab3a59d10bf4672bd053f0f72208f4d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/5ab3a59d10bf4672bd053f0f72208f4d 2024-11-10T04:37:09,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a37134c9de294561b10d0af09bc1f188 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/a37134c9de294561b10d0af09bc1f188 2024-11-10T04:37:09,701 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/recovered.edits/482.seqid, newMaxSeqId=482, maxSeqId=1 2024-11-10T04:37:09,702 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. 2024-11-10T04:37:09,702 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 6d2efd97fd4ccb14e877b384767c0ffc: Waiting for close lock at 1731213428299Running coprocessor pre-close hooks at 1731213428299Disabling compacts and flushes for region at 1731213428299Disabling writes for close at 1731213428299Obtaining lock to block concurrent updates at 1731213428299Preparing flush snapshotting stores in 6d2efd97fd4ccb14e877b384767c0ffc at 1731213428299Finished memstore snapshotting TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731213428299Flushing stores of TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc. at 1731213428300 (+1 ms)Flushing 6d2efd97fd4ccb14e877b384767c0ffc/A: creating writer at 1731213428300Flushing 6d2efd97fd4ccb14e877b384767c0ffc/A: appending metadata at 1731213428303 (+3 ms)Flushing 6d2efd97fd4ccb14e877b384767c0ffc/A: closing flushed file at 1731213428303Flushing 6d2efd97fd4ccb14e877b384767c0ffc/B: creating writer at 1731213428713 (+410 ms)Flushing 6d2efd97fd4ccb14e877b384767c0ffc/B: appending metadata at 1731213428716 (+3 ms)Flushing 6d2efd97fd4ccb14e877b384767c0ffc/B: closing flushed file at 1731213428716Flushing 6d2efd97fd4ccb14e877b384767c0ffc/C: creating writer at 1731213429126 (+410 ms)Flushing 6d2efd97fd4ccb14e877b384767c0ffc/C: appending metadata at 1731213429129 (+3 ms)Flushing 6d2efd97fd4ccb14e877b384767c0ffc/C: closing flushed file at 1731213429129Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@176f8615: reopening flushed file at 1731213429539 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72c9cf10: reopening flushed file at 1731213429544 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36cdd045: reopening flushed file at 1731213429548 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 6d2efd97fd4ccb14e877b384767c0ffc in 1255ms, sequenceid=479, compaction requested=false at 1731213429554 (+6 ms)Writing region close event to WAL at 1731213429697 (+143 ms)Running coprocessor post-close hooks at 1731213429702 (+5 ms)Closed at 1731213429702 2024-11-10T04:37:09,705 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:37:09,705 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6d2efd97fd4ccb14e877b384767c0ffc, regionState=CLOSED 2024-11-10T04:37:09,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:37:09,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-10T04:37:09,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 6d2efd97fd4ccb14e877b384767c0ffc, server=d1d9e3766cb1,45693,1731213391842 in 1.5670 sec 2024-11-10T04:37:09,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-10T04:37:09,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d2efd97fd4ccb14e877b384767c0ffc, UNASSIGN in 1.5740 sec 2024-11-10T04:37:09,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-10T04:37:09,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5810 sec 2024-11-10T04:37:09,716 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213429716"}]},"ts":"1731213429716"} 2024-11-10T04:37:09,718 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T04:37:09,718 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T04:37:09,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5970 sec 2024-11-10T04:37:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T04:37:10,272 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:10,275 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T04:37:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:10,281 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T04:37:10,282 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:10,287 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:37:10,290 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/recovered.edits] 2024-11-10T04:37:10,293 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/3cb5a4ac6c674672a34b94fb59287c79 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/3cb5a4ac6c674672a34b94fb59287c79 2024-11-10T04:37:10,295 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8a7291b84967434690bab122cff6d281 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/A/8a7291b84967434690bab122cff6d281 2024-11-10T04:37:10,297 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cd6959bb2757429f8800d8d5b565c79e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/cd6959bb2757429f8800d8d5b565c79e 2024-11-10T04:37:10,298 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/e3d1aab2d98148e4ba186dfcdb532b55 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/B/e3d1aab2d98148e4ba186dfcdb532b55 2024-11-10T04:37:10,301 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/665f3ccfdd08453195881fbbaa4e9b91 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/665f3ccfdd08453195881fbbaa4e9b91 2024-11-10T04:37:10,302 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/bfb032111a10461baefea5c0d46886f6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/C/bfb032111a10461baefea5c0d46886f6 2024-11-10T04:37:10,305 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/recovered.edits/482.seqid to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc/recovered.edits/482.seqid 2024-11-10T04:37:10,306 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d2efd97fd4ccb14e877b384767c0ffc 2024-11-10T04:37:10,306 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T04:37:10,311 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:10,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T04:37:10,320 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T04:37:10,349 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T04:37:10,350 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:10,350 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T04:37:10,350 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731213430350"}]},"ts":"9223372036854775807"} 2024-11-10T04:37:10,354 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T04:37:10,354 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6d2efd97fd4ccb14e877b384767c0ffc, NAME => 'TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T04:37:10,354 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T04:37:10,354 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731213430354"}]},"ts":"9223372036854775807"} 2024-11-10T04:37:10,357 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T04:37:10,358 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:10,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 82 msec 2024-11-10T04:37:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-10T04:37:10,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T04:37:10,392 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:10,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1218819243_22 at /127.0.0.1:42810 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;d1d9e3766cb1:45693-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 18) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6604 (was 7141) 2024-11-10T04:37:10,415 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=6603 2024-11-10T04:37:10,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T04:37:10,417 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:37:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:10,420 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:37:10,420 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:10,420 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-10T04:37:10,421 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:37:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T04:37:10,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741939_1115 (size=959) 2024-11-10T04:37:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T04:37:10,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T04:37:10,829 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8 2024-11-10T04:37:10,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741940_1116 (size=53) 2024-11-10T04:37:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T04:37:11,236 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:37:11,236 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing adb760d52bb12bfd79fa68b6162c27ba, disabling compactions & flushes 2024-11-10T04:37:11,236 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:11,236 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:11,236 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. after waiting 0 ms 2024-11-10T04:37:11,236 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:11,236 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:11,236 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for adb760d52bb12bfd79fa68b6162c27ba: Waiting for close lock at 1731213431236Disabling compacts and flushes for region at 1731213431236Disabling writes for close at 1731213431236Writing region close event to WAL at 1731213431236Closed at 1731213431236 2024-11-10T04:37:11,238 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:37:11,238 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731213431238"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731213431238"}]},"ts":"1731213431238"} 2024-11-10T04:37:11,240 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:37:11,241 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:37:11,241 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213431241"}]},"ts":"1731213431241"} 2024-11-10T04:37:11,242 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T04:37:11,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, ASSIGN}] 2024-11-10T04:37:11,243 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, ASSIGN 2024-11-10T04:37:11,244 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,45693,1731213391842; forceNewPlan=false, retain=false 2024-11-10T04:37:11,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=adb760d52bb12bfd79fa68b6162c27ba, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:11,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, ASSIGN because future has completed 2024-11-10T04:37:11,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:37:11,552 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:11,552 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:37:11,552 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,553 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:37:11,553 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,553 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,554 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,556 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adb760d52bb12bfd79fa68b6162c27ba columnFamilyName A 2024-11-10T04:37:11,557 DEBUG [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:11,557 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(327): Store=adb760d52bb12bfd79fa68b6162c27ba/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:37:11,557 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,558 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adb760d52bb12bfd79fa68b6162c27ba columnFamilyName B 2024-11-10T04:37:11,558 DEBUG [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:11,559 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(327): Store=adb760d52bb12bfd79fa68b6162c27ba/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:37:11,559 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,560 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adb760d52bb12bfd79fa68b6162c27ba columnFamilyName C 2024-11-10T04:37:11,560 DEBUG [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:11,561 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(327): Store=adb760d52bb12bfd79fa68b6162c27ba/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:37:11,561 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T04:37:11,561 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,562 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,563 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,563 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,563 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:37:11,564 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,566 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:37:11,567 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened adb760d52bb12bfd79fa68b6162c27ba; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73636067, jitterRate=0.097262904047966}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:37:11,567 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:11,567 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for adb760d52bb12bfd79fa68b6162c27ba: Running coprocessor pre-open hook at 1731213431553Writing region info on filesystem at 1731213431553Initializing all the Stores at 1731213431554 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213431554Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213431554Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213431554Cleaning up temporary data from old regions at 1731213431563 (+9 ms)Running coprocessor post-open hooks at 1731213431567 (+4 ms)Region opened successfully at 1731213431567 2024-11-10T04:37:11,568 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., pid=34, masterSystemTime=1731213431549 2024-11-10T04:37:11,570 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:11,570 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:11,571 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=adb760d52bb12bfd79fa68b6162c27ba, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:11,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:37:11,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-10T04:37:11,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 in 176 msec 2024-11-10T04:37:11,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-10T04:37:11,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, ASSIGN in 334 msec 2024-11-10T04:37:11,579 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:37:11,579 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213431579"}]},"ts":"1731213431579"} 2024-11-10T04:37:11,580 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T04:37:11,581 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:37:11,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-10T04:37:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-10T04:37:12,581 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:12,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:12,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:12,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:12,584 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3801661b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:12,586 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635a54b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:12,587 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:12,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:12,591 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:37:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,42929,1731213391190 2024-11-10T04:37:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@70e495a7 2024-11-10T04:37:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:37:12,593 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:37:12,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T04:37:12,598 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:37:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:12,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741941_1117 (size=995) 2024-11-10T04:37:13,017 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T04:37:13,018 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T04:37:13,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T04:37:13,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, REOPEN/MOVE}] 2024-11-10T04:37:13,029 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, REOPEN/MOVE 2024-11-10T04:37:13,030 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=adb760d52bb12bfd79fa68b6162c27ba, regionState=CLOSING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, REOPEN/MOVE because future has completed 2024-11-10T04:37:13,032 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T04:37:13,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:37:13,184 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,184 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T04:37:13,184 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing adb760d52bb12bfd79fa68b6162c27ba, disabling compactions & flushes 2024-11-10T04:37:13,184 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,184 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,184 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. after waiting 0 ms 2024-11-10T04:37:13,184 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,187 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T04:37:13,188 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,188 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for adb760d52bb12bfd79fa68b6162c27ba: Waiting for close lock at 1731213433184Running coprocessor pre-close hooks at 1731213433184Disabling compacts and flushes for region at 1731213433184Disabling writes for close at 1731213433184Writing region close event to WAL at 1731213433184Running coprocessor post-close hooks at 1731213433188 (+4 ms)Closed at 1731213433188 2024-11-10T04:37:13,188 WARN [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: adb760d52bb12bfd79fa68b6162c27ba to self. 2024-11-10T04:37:13,190 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,191 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=adb760d52bb12bfd79fa68b6162c27ba, regionState=CLOSED 2024-11-10T04:37:13,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:37:13,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-10T04:37:13,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 in 161 msec 2024-11-10T04:37:13,195 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, REOPEN/MOVE; state=CLOSED, location=d1d9e3766cb1,45693,1731213391842; forceNewPlan=false, retain=true 2024-11-10T04:37:13,346 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=adb760d52bb12bfd79fa68b6162c27ba, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, REOPEN/MOVE because future has completed 2024-11-10T04:37:13,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:37:13,503 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,503 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:37:13,503 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,503 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:37:13,503 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,504 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,508 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,513 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adb760d52bb12bfd79fa68b6162c27ba columnFamilyName A 2024-11-10T04:37:13,515 DEBUG [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:13,516 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(327): Store=adb760d52bb12bfd79fa68b6162c27ba/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:37:13,516 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,517 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adb760d52bb12bfd79fa68b6162c27ba columnFamilyName B 2024-11-10T04:37:13,517 DEBUG [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:13,517 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(327): Store=adb760d52bb12bfd79fa68b6162c27ba/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:37:13,517 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,518 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adb760d52bb12bfd79fa68b6162c27ba columnFamilyName C 2024-11-10T04:37:13,518 DEBUG [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:13,518 INFO [StoreOpener-adb760d52bb12bfd79fa68b6162c27ba-1 {}] regionserver.HStore(327): Store=adb760d52bb12bfd79fa68b6162c27ba/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:37:13,519 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,519 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,520 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,521 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,521 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,521 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:37:13,523 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,524 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened adb760d52bb12bfd79fa68b6162c27ba; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64855626, jitterRate=-0.033575862646102905}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:37:13,524 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,526 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for adb760d52bb12bfd79fa68b6162c27ba: Running coprocessor pre-open hook at 1731213433504Writing region info on filesystem at 1731213433504Initializing all the Stores at 1731213433504Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213433504Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213433505 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213433505Cleaning up temporary data from old regions at 1731213433521 (+16 ms)Running coprocessor post-open hooks at 1731213433524 (+3 ms)Region opened successfully at 1731213433525 (+1 ms) 2024-11-10T04:37:13,526 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., pid=39, masterSystemTime=1731213433500 2024-11-10T04:37:13,528 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,528 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,529 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=adb760d52bb12bfd79fa68b6162c27ba, regionState=OPEN, openSeqNum=5, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:37:13,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-10T04:37:13,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 in 183 msec 2024-11-10T04:37:13,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-10T04:37:13,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, REOPEN/MOVE in 505 msec 2024-11-10T04:37:13,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-10T04:37:13,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-10T04:37:13,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-10T04:37:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-10T04:37:13,544 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:13,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:13,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:13,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@232796c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:13,547 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:13,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e8013a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:13,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:13,549 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aff2db3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:13,551 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:13,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67042676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:13,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:13,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18877e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:13,555 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8e8546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:13,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:13,557 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b13d34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:13,559 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c8a60d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:13,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:13,561 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f57d39e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:13,562 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:13,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25786e22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:13,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:13,564 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d8f127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:13,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9b8755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:13,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:13,568 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9cb2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:13,569 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c88dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:13,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:13,571 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758782aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:13,572 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c602edf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:13,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:13,574 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12262db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:13,575 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677da74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:13,576 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:13,576 DEBUG [Thread-606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:13,576 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:13,577 DEBUG [Thread-619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:13,577 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:13,577 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:13,577 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-10T04:37:13,578 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:13,579 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:13,579 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:13,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:13,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:13,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:13,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T04:37:13,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:13,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:13,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:13,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:13,581 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:13,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:13,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:13,582 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:13,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:13,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:13,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:37:13,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:37:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:37:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:37:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:37:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:37:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:37:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:37:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:37:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:13,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:13,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:37:13,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a566d05e4b42442baf4d02d795edeaf7_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213433596/Put/seqid=0 2024-11-10T04:37:13,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741942_1118 (size=12154) 2024-11-10T04:37:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213493655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213493657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213493658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213493661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213493663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T04:37:13,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:13,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:13,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:13,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213493771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213493771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213493781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213493781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213493781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:13,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:13,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:13,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:13,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T04:37:13,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213493982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213493982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213493991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213493992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213493992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:13,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:14,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:14,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:14,068 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a566d05e4b42442baf4d02d795edeaf7_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a566d05e4b42442baf4d02d795edeaf7_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:14,071 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/76689d4e4aeb4d7e8440ffa7c61caebd, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:14,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/76689d4e4aeb4d7e8440ffa7c61caebd is 175, key is test_row_0/A:col10/1731213433596/Put/seqid=0 2024-11-10T04:37:14,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741943_1119 (size=30955) 2024-11-10T04:37:14,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T04:37:14,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213494301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213494301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213494301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213494301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213494302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:14,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:14,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,488 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/76689d4e4aeb4d7e8440ffa7c61caebd 2024-11-10T04:37:14,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/c23739ae3cd344ebb8d732c027968373 is 50, key is test_row_0/B:col10/1731213433596/Put/seqid=0 2024-11-10T04:37:14,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741944_1120 (size=12001) 2024-11-10T04:37:14,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T04:37:14,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213494811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213494812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213494812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213494812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213494812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:14,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/c23739ae3cd344ebb8d732c027968373 2024-11-10T04:37:14,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/eae0b7f721be41ca8e067bc385ac39e3 is 50, key is test_row_0/C:col10/1731213433596/Put/seqid=0 2024-11-10T04:37:14,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741945_1121 (size=12001) 2024-11-10T04:37:14,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:14,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:15,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:15,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:15,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:15,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:15,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:15,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:15,237 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T04:37:15,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:15,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:15,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:15,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:15,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:15,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:15,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/eae0b7f721be41ca8e067bc385ac39e3 2024-11-10T04:37:15,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/76689d4e4aeb4d7e8440ffa7c61caebd as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/76689d4e4aeb4d7e8440ffa7c61caebd 2024-11-10T04:37:15,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/76689d4e4aeb4d7e8440ffa7c61caebd, entries=150, sequenceid=15, filesize=30.2 K 2024-11-10T04:37:15,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/c23739ae3cd344ebb8d732c027968373 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/c23739ae3cd344ebb8d732c027968373 2024-11-10T04:37:15,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/c23739ae3cd344ebb8d732c027968373, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T04:37:15,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/eae0b7f721be41ca8e067bc385ac39e3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eae0b7f721be41ca8e067bc385ac39e3 2024-11-10T04:37:15,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eae0b7f721be41ca8e067bc385ac39e3, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T04:37:15,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for adb760d52bb12bfd79fa68b6162c27ba in 1801ms, sequenceid=15, compaction requested=false 2024-11-10T04:37:15,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:15,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T04:37:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:15,442 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:37:15,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d290b655b2074c30b595a130fa980f78_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213433661/Put/seqid=0 2024-11-10T04:37:15,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741946_1122 (size=12154) 2024-11-10T04:37:15,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:15,497 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d290b655b2074c30b595a130fa980f78_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d290b655b2074c30b595a130fa980f78_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:15,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/92c41da7e1d24a9686e82c96e2aa43da, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:15,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/92c41da7e1d24a9686e82c96e2aa43da is 175, key is test_row_0/A:col10/1731213433661/Put/seqid=0 2024-11-10T04:37:15,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741947_1123 (size=30955) 2024-11-10T04:37:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T04:37:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:15,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:15,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213495861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213495861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:15,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213495868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:15,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213495870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213495870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:15,947 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/92c41da7e1d24a9686e82c96e2aa43da 2024-11-10T04:37:15,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/29e4e48246634d7c80ffa7b85f5b7020 is 50, key is test_row_0/B:col10/1731213433661/Put/seqid=0 2024-11-10T04:37:15,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741948_1124 (size=12001) 2024-11-10T04:37:15,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213495981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213495981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213495981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213495982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:15,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213495982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,054 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T04:37:16,054 INFO [master/d1d9e3766cb1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T04:37:16,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213496191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213496191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213496192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213496192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213496192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,362 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/29e4e48246634d7c80ffa7b85f5b7020 2024-11-10T04:37:16,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/2cd5c46a76cb45bfbefd646107564f24 is 50, key is test_row_0/C:col10/1731213433661/Put/seqid=0 2024-11-10T04:37:16,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741949_1125 (size=12001) 2024-11-10T04:37:16,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213496501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213496501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213496502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213496502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213496502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:16,776 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/2cd5c46a76cb45bfbefd646107564f24 2024-11-10T04:37:16,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/92c41da7e1d24a9686e82c96e2aa43da as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/92c41da7e1d24a9686e82c96e2aa43da 2024-11-10T04:37:16,787 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/92c41da7e1d24a9686e82c96e2aa43da, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T04:37:16,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/29e4e48246634d7c80ffa7b85f5b7020 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/29e4e48246634d7c80ffa7b85f5b7020 2024-11-10T04:37:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,794 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/29e4e48246634d7c80ffa7b85f5b7020, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T04:37:16,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T04:37:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/2cd5c46a76cb45bfbefd646107564f24 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/2cd5c46a76cb45bfbefd646107564f24 2024-11-10T04:37:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,806 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/2cd5c46a76cb45bfbefd646107564f24, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T04:37:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,807 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for adb760d52bb12bfd79fa68b6162c27ba in 1365ms, sequenceid=40, compaction requested=false 2024-11-10T04:37:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:16,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:16,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-10T04:37:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-10T04:37:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-10T04:37:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2270 sec 2024-11-10T04:37:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.2360 sec 2024-11-10T04:37:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:17,040 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T04:37:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107bd0a7e6067440af8b9ff6be4d3c3642_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_1/A:col10/1731213437038/Put/seqid=0 2024-11-10T04:37:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741950_1126 (size=14594) 2024-11-10T04:37:17,063 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,068 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107bd0a7e6067440af8b9ff6be4d3c3642_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107bd0a7e6067440af8b9ff6be4d3c3642_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:17,069 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/922008d41cc64a169224b76e2f39e776, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:17,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/922008d41cc64a169224b76e2f39e776 is 175, key is test_row_1/A:col10/1731213437038/Put/seqid=0 2024-11-10T04:37:17,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741951_1127 (size=39545) 2024-11-10T04:37:17,084 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/922008d41cc64a169224b76e2f39e776 2024-11-10T04:37:17,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/28fd76aca35c47c59bddfeed2b6dbd76 is 50, key is test_row_1/B:col10/1731213437038/Put/seqid=0 2024-11-10T04:37:17,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741952_1128 (size=9657) 2024-11-10T04:37:17,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213497131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213497132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213497134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213497137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213497139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213497241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213497242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213497242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213497242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213497251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213497451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213497452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213497452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213497452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213497461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/28fd76aca35c47c59bddfeed2b6dbd76 2024-11-10T04:37:17,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/6afdfed40c844f9faf3edd7e78771cfe is 50, key is test_row_1/C:col10/1731213437038/Put/seqid=0 2024-11-10T04:37:17,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741953_1129 (size=9657) 2024-11-10T04:37:17,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/6afdfed40c844f9faf3edd7e78771cfe 2024-11-10T04:37:17,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/922008d41cc64a169224b76e2f39e776 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/922008d41cc64a169224b76e2f39e776 2024-11-10T04:37:17,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/922008d41cc64a169224b76e2f39e776, entries=200, sequenceid=54, filesize=38.6 K 2024-11-10T04:37:17,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/28fd76aca35c47c59bddfeed2b6dbd76 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/28fd76aca35c47c59bddfeed2b6dbd76 2024-11-10T04:37:17,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/28fd76aca35c47c59bddfeed2b6dbd76, entries=100, sequenceid=54, filesize=9.4 K 2024-11-10T04:37:17,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/6afdfed40c844f9faf3edd7e78771cfe as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6afdfed40c844f9faf3edd7e78771cfe 2024-11-10T04:37:17,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6afdfed40c844f9faf3edd7e78771cfe, entries=100, sequenceid=54, filesize=9.4 K 2024-11-10T04:37:17,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for adb760d52bb12bfd79fa68b6162c27ba in 509ms, sequenceid=54, compaction requested=true 2024-11-10T04:37:17,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:17,550 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:17,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:17,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:17,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:17,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:17,551 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:17,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:17,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:17,552 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:17,552 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:17,552 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/A is initiating minor compaction (all files) 2024-11-10T04:37:17,552 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/B is initiating minor compaction (all files) 2024-11-10T04:37:17,552 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/A in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:17,552 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/B in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:17,552 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/76689d4e4aeb4d7e8440ffa7c61caebd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/92c41da7e1d24a9686e82c96e2aa43da, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/922008d41cc64a169224b76e2f39e776] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=99.1 K 2024-11-10T04:37:17,552 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/c23739ae3cd344ebb8d732c027968373, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/29e4e48246634d7c80ffa7b85f5b7020, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/28fd76aca35c47c59bddfeed2b6dbd76] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=32.9 K 2024-11-10T04:37:17,552 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:17,553 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/76689d4e4aeb4d7e8440ffa7c61caebd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/92c41da7e1d24a9686e82c96e2aa43da, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/922008d41cc64a169224b76e2f39e776] 2024-11-10T04:37:17,553 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76689d4e4aeb4d7e8440ffa7c61caebd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731213433589 2024-11-10T04:37:17,553 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c23739ae3cd344ebb8d732c027968373, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731213433589 2024-11-10T04:37:17,554 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c41da7e1d24a9686e82c96e2aa43da, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731213433656 2024-11-10T04:37:17,554 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 29e4e48246634d7c80ffa7b85f5b7020, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731213433656 2024-11-10T04:37:17,554 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 922008d41cc64a169224b76e2f39e776, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731213435867 2024-11-10T04:37:17,555 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 28fd76aca35c47c59bddfeed2b6dbd76, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731213435867 2024-11-10T04:37:17,565 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:17,565 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:17,566 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/6f2198e0e29a481f8c128d3f8ae88c23 is 50, key is test_row_0/B:col10/1731213433661/Put/seqid=0 2024-11-10T04:37:17,573 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110ff2a1313a4cb49658732712347b9904e_adb760d52bb12bfd79fa68b6162c27ba store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:17,576 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110ff2a1313a4cb49658732712347b9904e_adb760d52bb12bfd79fa68b6162c27ba, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:17,576 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ff2a1313a4cb49658732712347b9904e_adb760d52bb12bfd79fa68b6162c27ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:17,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741954_1130 (size=12104) 2024-11-10T04:37:17,592 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/6f2198e0e29a481f8c128d3f8ae88c23 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/6f2198e0e29a481f8c128d3f8ae88c23 2024-11-10T04:37:17,598 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/B of adb760d52bb12bfd79fa68b6162c27ba into 6f2198e0e29a481f8c128d3f8ae88c23(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:17,599 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:17,599 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/B, priority=13, startTime=1731213437550; duration=0sec 2024-11-10T04:37:17,599 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:17,599 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:B 2024-11-10T04:37:17,599 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:17,600 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:17,600 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/C is initiating minor compaction (all files) 2024-11-10T04:37:17,600 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/C in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:17,601 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eae0b7f721be41ca8e067bc385ac39e3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/2cd5c46a76cb45bfbefd646107564f24, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6afdfed40c844f9faf3edd7e78771cfe] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=32.9 K 2024-11-10T04:37:17,601 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting eae0b7f721be41ca8e067bc385ac39e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731213433589 2024-11-10T04:37:17,601 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd5c46a76cb45bfbefd646107564f24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731213433656 2024-11-10T04:37:17,602 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6afdfed40c844f9faf3edd7e78771cfe, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731213435867 2024-11-10T04:37:17,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741955_1131 (size=4469) 2024-11-10T04:37:17,611 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#C#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:17,612 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/25fe1f979b1b4ae9a987df12aabf5e03 is 50, key is test_row_0/C:col10/1731213433661/Put/seqid=0 2024-11-10T04:37:17,615 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#A#compaction#112 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:17,615 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/aeae44ed05454996b9e836d70cf1e2e7 is 175, key is test_row_0/A:col10/1731213433661/Put/seqid=0 2024-11-10T04:37:17,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741957_1133 (size=31165) 2024-11-10T04:37:17,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741956_1132 (size=12104) 2024-11-10T04:37:17,650 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/aeae44ed05454996b9e836d70cf1e2e7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/aeae44ed05454996b9e836d70cf1e2e7 2024-11-10T04:37:17,659 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/A of adb760d52bb12bfd79fa68b6162c27ba into aeae44ed05454996b9e836d70cf1e2e7(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:17,659 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:17,659 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/A, priority=13, startTime=1731213437550; duration=0sec 2024-11-10T04:37:17,659 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:17,659 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:A 2024-11-10T04:37:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T04:37:17,741 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:17,742 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-10T04:37:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T04:37:17,744 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:17,745 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:17,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:17,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:17,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T04:37:17,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c5254bec3ed8494097837ccae3b7af09_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213437133/Put/seqid=0 2024-11-10T04:37:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213497775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213497775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213497777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213497779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213497780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741958_1134 (size=14594) 2024-11-10T04:37:17,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:17,803 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c5254bec3ed8494097837ccae3b7af09_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c5254bec3ed8494097837ccae3b7af09_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:17,806 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/7329709d3d8a41c5b345494f01aa2edc, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:17,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/7329709d3d8a41c5b345494f01aa2edc is 175, key is test_row_0/A:col10/1731213437133/Put/seqid=0 2024-11-10T04:37:17,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741959_1135 (size=39549) 2024-11-10T04:37:17,827 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/7329709d3d8a41c5b345494f01aa2edc 2024-11-10T04:37:17,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/4f07209a3c3747b1a64535490ab0984c is 50, key is test_row_0/B:col10/1731213437133/Put/seqid=0 2024-11-10T04:37:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T04:37:17,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741960_1136 (size=12001) 2024-11-10T04:37:17,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/4f07209a3c3747b1a64535490ab0984c 2024-11-10T04:37:17,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/8bde4762e8c847419f8f8d18e1175fd6 is 50, key is test_row_0/C:col10/1731213437133/Put/seqid=0 2024-11-10T04:37:17,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741961_1137 (size=12001) 2024-11-10T04:37:17,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/8bde4762e8c847419f8f8d18e1175fd6 2024-11-10T04:37:17,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213497892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213497892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213497892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213497893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213497894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/7329709d3d8a41c5b345494f01aa2edc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7329709d3d8a41c5b345494f01aa2edc 2024-11-10T04:37:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:17,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T04:37:17,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:17,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:17,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:17,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:17,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:17,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7329709d3d8a41c5b345494f01aa2edc, entries=200, sequenceid=80, filesize=38.6 K 2024-11-10T04:37:17,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/4f07209a3c3747b1a64535490ab0984c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4f07209a3c3747b1a64535490ab0984c 2024-11-10T04:37:17,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4f07209a3c3747b1a64535490ab0984c, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T04:37:17,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/8bde4762e8c847419f8f8d18e1175fd6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8bde4762e8c847419f8f8d18e1175fd6 2024-11-10T04:37:17,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8bde4762e8c847419f8f8d18e1175fd6, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T04:37:17,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for adb760d52bb12bfd79fa68b6162c27ba in 161ms, sequenceid=80, compaction requested=false 2024-11-10T04:37:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:18,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T04:37:18,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:18,053 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:37:18,056 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/25fe1f979b1b4ae9a987df12aabf5e03 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/25fe1f979b1b4ae9a987df12aabf5e03 2024-11-10T04:37:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T04:37:18,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103db23dc8661344e5afb3a604351212f9_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213437771/Put/seqid=0 2024-11-10T04:37:18,068 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/C of adb760d52bb12bfd79fa68b6162c27ba into 25fe1f979b1b4ae9a987df12aabf5e03(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:18,068 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:18,068 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/C, priority=13, startTime=1731213437551; duration=0sec 2024-11-10T04:37:18,068 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:18,068 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:C 2024-11-10T04:37:18,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741962_1138 (size=12154) 2024-11-10T04:37:18,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:18,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213498133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213498134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213498137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213498137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213498137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213498241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213498241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213498252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213498252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213498252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T04:37:18,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213498452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213498452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213498461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213498461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213498461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:18,479 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103db23dc8661344e5afb3a604351212f9_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103db23dc8661344e5afb3a604351212f9_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/62cb3f394e1a4b4ab794aa984e5b2ade, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:18,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/62cb3f394e1a4b4ab794aa984e5b2ade is 175, key is test_row_0/A:col10/1731213437771/Put/seqid=0 2024-11-10T04:37:18,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741963_1139 (size=30955) 2024-11-10T04:37:18,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213498762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213498771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213498771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213498771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213498771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T04:37:18,886 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/62cb3f394e1a4b4ab794aa984e5b2ade 2024-11-10T04:37:18,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/13def43cee114737bd7ec9dc4467c12d is 50, key is test_row_0/B:col10/1731213437771/Put/seqid=0 2024-11-10T04:37:18,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741964_1140 (size=12001) 2024-11-10T04:37:19,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213499271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:19,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213499281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:19,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213499281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213499281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213499281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:19,304 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/13def43cee114737bd7ec9dc4467c12d 2024-11-10T04:37:19,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/ca4955d362c74c9e863140855bbb914b is 50, key is test_row_0/C:col10/1731213437771/Put/seqid=0 2024-11-10T04:37:19,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741965_1141 (size=12001) 2024-11-10T04:37:19,726 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/ca4955d362c74c9e863140855bbb914b 2024-11-10T04:37:19,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/62cb3f394e1a4b4ab794aa984e5b2ade as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/62cb3f394e1a4b4ab794aa984e5b2ade 2024-11-10T04:37:19,735 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/62cb3f394e1a4b4ab794aa984e5b2ade, entries=150, sequenceid=92, filesize=30.2 K 2024-11-10T04:37:19,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/13def43cee114737bd7ec9dc4467c12d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/13def43cee114737bd7ec9dc4467c12d 2024-11-10T04:37:19,741 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/13def43cee114737bd7ec9dc4467c12d, entries=150, sequenceid=92, filesize=11.7 K 2024-11-10T04:37:19,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/ca4955d362c74c9e863140855bbb914b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/ca4955d362c74c9e863140855bbb914b 2024-11-10T04:37:19,749 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/ca4955d362c74c9e863140855bbb914b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-10T04:37:19,750 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for adb760d52bb12bfd79fa68b6162c27ba in 1697ms, sequenceid=92, compaction requested=true 2024-11-10T04:37:19,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:19,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:19,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-10T04:37:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-10T04:37:19,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-10T04:37:19,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0080 sec 2024-11-10T04:37:19,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.0140 sec 2024-11-10T04:37:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T04:37:19,891 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:19,892 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T04:37:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T04:37:19,894 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:19,894 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:19,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T04:37:20,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T04:37:20,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:20,047 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:37:20,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cd9931747ac24fafb7ce9644aa533d00_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213438134/Put/seqid=0 2024-11-10T04:37:20,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741966_1142 (size=12154) 2024-11-10T04:37:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T04:37:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:20,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213500303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213500304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213500304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213500305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213500309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213500411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213500421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213500421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213500421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213500422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:20,466 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cd9931747ac24fafb7ce9644aa533d00_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cd9931747ac24fafb7ce9644aa533d00_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:20,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9624207fa92345088913d71c41584b5d, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9624207fa92345088913d71c41584b5d is 175, key is test_row_0/A:col10/1731213438134/Put/seqid=0 2024-11-10T04:37:20,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741967_1143 (size=30955) 2024-11-10T04:37:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T04:37:20,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213500621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213500631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213500631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213500631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213500631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,873 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9624207fa92345088913d71c41584b5d 2024-11-10T04:37:20,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/d7966afbd0bf4f3ea3489e91d1ac4e47 is 50, key is test_row_0/B:col10/1731213438134/Put/seqid=0 2024-11-10T04:37:20,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741968_1144 (size=12001) 2024-11-10T04:37:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213500931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213500942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213500942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213500942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213500942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T04:37:21,295 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/d7966afbd0bf4f3ea3489e91d1ac4e47 2024-11-10T04:37:21,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/d912af78e4cc4915800b90d3ac2028dc is 50, key is test_row_0/C:col10/1731213438134/Put/seqid=0 2024-11-10T04:37:21,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741969_1145 (size=12001) 2024-11-10T04:37:21,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213501441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:21,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213501451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213501451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:21,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213501451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213501451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:21,709 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/d912af78e4cc4915800b90d3ac2028dc 2024-11-10T04:37:21,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9624207fa92345088913d71c41584b5d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9624207fa92345088913d71c41584b5d 2024-11-10T04:37:21,719 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9624207fa92345088913d71c41584b5d, entries=150, sequenceid=118, filesize=30.2 K 2024-11-10T04:37:21,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/d7966afbd0bf4f3ea3489e91d1ac4e47 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7966afbd0bf4f3ea3489e91d1ac4e47 2024-11-10T04:37:21,724 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7966afbd0bf4f3ea3489e91d1ac4e47, entries=150, sequenceid=118, filesize=11.7 K 2024-11-10T04:37:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/d912af78e4cc4915800b90d3ac2028dc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d912af78e4cc4915800b90d3ac2028dc 2024-11-10T04:37:21,729 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d912af78e4cc4915800b90d3ac2028dc, entries=150, sequenceid=118, filesize=11.7 K 2024-11-10T04:37:21,730 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for adb760d52bb12bfd79fa68b6162c27ba in 1683ms, sequenceid=118, compaction requested=true 2024-11-10T04:37:21,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:21,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:21,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T04:37:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T04:37:21,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T04:37:21,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8370 sec 2024-11-10T04:37:21,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8420 sec 2024-11-10T04:37:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T04:37:22,042 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:22,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T04:37:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T04:37:22,045 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:22,046 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:22,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T04:37:22,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T04:37:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:22,199 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:37:22,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104e99887c6b144ef1b78e3f6d949a9a02_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213440308/Put/seqid=0 2024-11-10T04:37:22,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741970_1146 (size=12154) 2024-11-10T04:37:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T04:37:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:22,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213502490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213502490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213502492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213502498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213502500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:22,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213502611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213502612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213502612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213502612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213502612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,617 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104e99887c6b144ef1b78e3f6d949a9a02_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e99887c6b144ef1b78e3f6d949a9a02_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/6814e12ba0a4485e8de4811d027e77dc, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/6814e12ba0a4485e8de4811d027e77dc is 175, key is test_row_0/A:col10/1731213440308/Put/seqid=0 2024-11-10T04:37:22,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741971_1147 (size=30955) 2024-11-10T04:37:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T04:37:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213502821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213502821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213502821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213502821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213502821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:22,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,024 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/6814e12ba0a4485e8de4811d027e77dc 2024-11-10T04:37:23,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/ec020211764448939ed2e5546aef2919 is 50, key is test_row_0/B:col10/1731213440308/Put/seqid=0 2024-11-10T04:37:23,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741972_1148 (size=12001) 2024-11-10T04:37:23,038 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/ec020211764448939ed2e5546aef2919 2024-11-10T04:37:23,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/f1f75e1ed4db4a68b9443ccd66a79b6b is 50, key is test_row_0/C:col10/1731213440308/Put/seqid=0 2024-11-10T04:37:23,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741973_1149 (size=12001) 2024-11-10T04:37:23,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213503132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213503132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213503132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213503141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213503141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T04:37:23,452 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/f1f75e1ed4db4a68b9443ccd66a79b6b 2024-11-10T04:37:23,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/6814e12ba0a4485e8de4811d027e77dc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/6814e12ba0a4485e8de4811d027e77dc 2024-11-10T04:37:23,463 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/6814e12ba0a4485e8de4811d027e77dc, entries=150, sequenceid=129, filesize=30.2 K 2024-11-10T04:37:23,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/ec020211764448939ed2e5546aef2919 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/ec020211764448939ed2e5546aef2919 2024-11-10T04:37:23,469 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/ec020211764448939ed2e5546aef2919, entries=150, sequenceid=129, filesize=11.7 K 2024-11-10T04:37:23,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/f1f75e1ed4db4a68b9443ccd66a79b6b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f1f75e1ed4db4a68b9443ccd66a79b6b 2024-11-10T04:37:23,475 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f1f75e1ed4db4a68b9443ccd66a79b6b, entries=150, sequenceid=129, filesize=11.7 K 2024-11-10T04:37:23,477 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for adb760d52bb12bfd79fa68b6162c27ba in 1279ms, sequenceid=129, compaction requested=true 2024-11-10T04:37:23,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:23,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:23,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T04:37:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T04:37:23,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T04:37:23,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4330 sec 2024-11-10T04:37:23,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4410 sec 2024-11-10T04:37:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:23,645 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:23,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:37:23,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a5cc7485abfd4588830b46d540703f3f_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213442489/Put/seqid=0 2024-11-10T04:37:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213503652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213503652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213503652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213503653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213503659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741974_1150 (size=14794) 2024-11-10T04:37:23,667 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:23,671 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a5cc7485abfd4588830b46d540703f3f_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a5cc7485abfd4588830b46d540703f3f_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:23,672 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9945c113b79a41e69a898e1841a09e3a, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:23,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9945c113b79a41e69a898e1841a09e3a is 175, key is test_row_0/A:col10/1731213442489/Put/seqid=0 2024-11-10T04:37:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741975_1151 (size=39749) 2024-11-10T04:37:23,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213503771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213503771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213503771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213503981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213503982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213503982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:24,078 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9945c113b79a41e69a898e1841a09e3a 2024-11-10T04:37:24,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/e4b17652be5e43349e30770ed0e3489d is 50, key is test_row_0/B:col10/1731213442489/Put/seqid=0 2024-11-10T04:37:24,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741976_1152 (size=12151) 2024-11-10T04:37:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T04:37:24,191 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:24,192 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-10T04:37:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T04:37:24,194 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:24,194 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:24,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213504291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213504291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213504291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T04:37:24,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T04:37:24,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:24,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/e4b17652be5e43349e30770ed0e3489d 2024-11-10T04:37:24,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T04:37:24,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:24,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/a4f691b41715444bbcd6ab9d31ff6fc4 is 50, key is test_row_0/C:col10/1731213442489/Put/seqid=0 2024-11-10T04:37:24,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741977_1153 (size=12151) 2024-11-10T04:37:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T04:37:24,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T04:37:24,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:24,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213504671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:24,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213504671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:24,675 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:24,675 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:24,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:24,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213504801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213504801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:24,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213504801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:24,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:24,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T04:37:24,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:24,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T04:37:24,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/a4f691b41715444bbcd6ab9d31ff6fc4 2024-11-10T04:37:24,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9945c113b79a41e69a898e1841a09e3a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9945c113b79a41e69a898e1841a09e3a 2024-11-10T04:37:24,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9945c113b79a41e69a898e1841a09e3a, entries=200, sequenceid=156, filesize=38.8 K 2024-11-10T04:37:24,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/e4b17652be5e43349e30770ed0e3489d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e4b17652be5e43349e30770ed0e3489d 2024-11-10T04:37:24,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e4b17652be5e43349e30770ed0e3489d, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T04:37:24,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/a4f691b41715444bbcd6ab9d31ff6fc4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/a4f691b41715444bbcd6ab9d31ff6fc4 2024-11-10T04:37:24,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/a4f691b41715444bbcd6ab9d31ff6fc4, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T04:37:24,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for adb760d52bb12bfd79fa68b6162c27ba in 1286ms, sequenceid=156, compaction requested=true 2024-11-10T04:37:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:24,932 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T04:37:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:24,932 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T04:37:24,934 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 203328 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T04:37:24,934 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72259 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T04:37:24,934 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/B is initiating minor compaction (all files) 2024-11-10T04:37:24,934 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/A is initiating minor compaction (all files) 2024-11-10T04:37:24,934 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/B in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,934 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/A in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,934 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/6f2198e0e29a481f8c128d3f8ae88c23, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4f07209a3c3747b1a64535490ab0984c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/13def43cee114737bd7ec9dc4467c12d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7966afbd0bf4f3ea3489e91d1ac4e47, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/ec020211764448939ed2e5546aef2919, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e4b17652be5e43349e30770ed0e3489d] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=70.6 K 2024-11-10T04:37:24,934 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/aeae44ed05454996b9e836d70cf1e2e7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7329709d3d8a41c5b345494f01aa2edc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/62cb3f394e1a4b4ab794aa984e5b2ade, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9624207fa92345088913d71c41584b5d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/6814e12ba0a4485e8de4811d027e77dc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9945c113b79a41e69a898e1841a09e3a] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=198.6 K 2024-11-10T04:37:24,934 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,935 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/aeae44ed05454996b9e836d70cf1e2e7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7329709d3d8a41c5b345494f01aa2edc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/62cb3f394e1a4b4ab794aa984e5b2ade, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9624207fa92345088913d71c41584b5d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/6814e12ba0a4485e8de4811d027e77dc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9945c113b79a41e69a898e1841a09e3a] 2024-11-10T04:37:24,936 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f2198e0e29a481f8c128d3f8ae88c23, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731213433661 2024-11-10T04:37:24,936 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeae44ed05454996b9e836d70cf1e2e7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731213433661 2024-11-10T04:37:24,936 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f07209a3c3747b1a64535490ab0984c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731213437133 2024-11-10T04:37:24,936 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7329709d3d8a41c5b345494f01aa2edc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731213437132 2024-11-10T04:37:24,937 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62cb3f394e1a4b4ab794aa984e5b2ade, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731213437771 2024-11-10T04:37:24,937 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 13def43cee114737bd7ec9dc4467c12d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731213437771 2024-11-10T04:37:24,937 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9624207fa92345088913d71c41584b5d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731213438131 2024-11-10T04:37:24,938 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d7966afbd0bf4f3ea3489e91d1ac4e47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731213438131 2024-11-10T04:37:24,939 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ec020211764448939ed2e5546aef2919, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731213440300 2024-11-10T04:37:24,939 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6814e12ba0a4485e8de4811d027e77dc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731213440300 2024-11-10T04:37:24,939 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b17652be5e43349e30770ed0e3489d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731213442489 2024-11-10T04:37:24,939 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9945c113b79a41e69a898e1841a09e3a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731213442489 2024-11-10T04:37:24,954 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:24,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T04:37:24,961 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#B#compaction#130 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:24,961 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-10T04:37:24,962 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/b19625148d1c40788e11c700756b16a6 is 50, key is test_row_0/B:col10/1731213442489/Put/seqid=0 2024-11-10T04:37:24,963 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110185cf62130fa4006a14b4145e68f835d_adb760d52bb12bfd79fa68b6162c27ba store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:24,967 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110185cf62130fa4006a14b4145e68f835d_adb760d52bb12bfd79fa68b6162c27ba, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:24,967 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110185cf62130fa4006a14b4145e68f835d_adb760d52bb12bfd79fa68b6162c27ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:24,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741978_1154 (size=12459) 2024-11-10T04:37:24,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110050879b874b84f0a9fa0ba63ac870892_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213443658/Put/seqid=0 2024-11-10T04:37:24,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741979_1155 (size=4469) 2024-11-10T04:37:24,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741980_1156 (size=12304) 2024-11-10T04:37:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T04:37:25,388 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#A#compaction#129 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:25,389 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/2c9365fcb62f4a6c9342c3be299dfe46 is 175, key is test_row_0/A:col10/1731213442489/Put/seqid=0 2024-11-10T04:37:25,392 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/b19625148d1c40788e11c700756b16a6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/b19625148d1c40788e11c700756b16a6 2024-11-10T04:37:25,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741981_1157 (size=31413) 2024-11-10T04:37:25,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:25,400 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/B of adb760d52bb12bfd79fa68b6162c27ba into b19625148d1c40788e11c700756b16a6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:25,400 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:25,400 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/B, priority=10, startTime=1731213444932; duration=0sec 2024-11-10T04:37:25,400 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:25,400 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:B 2024-11-10T04:37:25,400 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-10T04:37:25,401 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110050879b874b84f0a9fa0ba63ac870892_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110050879b874b84f0a9fa0ba63ac870892_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:25,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/5d94ac9598f446f0849af7a60fc1fdd3, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:25,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/5d94ac9598f446f0849af7a60fc1fdd3 is 175, key is test_row_0/A:col10/1731213443658/Put/seqid=0 2024-11-10T04:37:25,405 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72259 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-10T04:37:25,405 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/C is initiating minor compaction (all files) 2024-11-10T04:37:25,405 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/C in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:25,405 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/25fe1f979b1b4ae9a987df12aabf5e03, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8bde4762e8c847419f8f8d18e1175fd6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/ca4955d362c74c9e863140855bbb914b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d912af78e4cc4915800b90d3ac2028dc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f1f75e1ed4db4a68b9443ccd66a79b6b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/a4f691b41715444bbcd6ab9d31ff6fc4] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=70.6 K 2024-11-10T04:37:25,405 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 25fe1f979b1b4ae9a987df12aabf5e03, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731213433661 2024-11-10T04:37:25,406 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bde4762e8c847419f8f8d18e1175fd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731213437133 2024-11-10T04:37:25,406 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4955d362c74c9e863140855bbb914b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731213437771 2024-11-10T04:37:25,407 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d912af78e4cc4915800b90d3ac2028dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731213438131 2024-11-10T04:37:25,407 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f75e1ed4db4a68b9443ccd66a79b6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731213440300 2024-11-10T04:37:25,407 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f691b41715444bbcd6ab9d31ff6fc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731213442489 2024-11-10T04:37:25,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741982_1158 (size=31105) 2024-11-10T04:37:25,410 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/5d94ac9598f446f0849af7a60fc1fdd3 2024-11-10T04:37:25,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/4136451a59f745398e0f11369e27453f is 50, key is test_row_0/B:col10/1731213443658/Put/seqid=0 2024-11-10T04:37:25,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741983_1159 (size=12151) 2024-11-10T04:37:25,427 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#C#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:25,428 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/f7bd24dda3854068b680a09589e9922a is 50, key is test_row_0/C:col10/1731213442489/Put/seqid=0 2024-11-10T04:37:25,430 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/4136451a59f745398e0f11369e27453f 2024-11-10T04:37:25,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/c3cc2e4de6b24a76abde6bd2a7230691 is 50, key is test_row_0/C:col10/1731213443658/Put/seqid=0 2024-11-10T04:37:25,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741984_1160 (size=12459) 2024-11-10T04:37:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741985_1161 (size=12151) 2024-11-10T04:37:25,802 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/2c9365fcb62f4a6c9342c3be299dfe46 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/2c9365fcb62f4a6c9342c3be299dfe46 2024-11-10T04:37:25,808 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/A of adb760d52bb12bfd79fa68b6162c27ba into 2c9365fcb62f4a6c9342c3be299dfe46(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:25,808 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:25,808 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/A, priority=10, startTime=1731213444932; duration=0sec 2024-11-10T04:37:25,808 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:25,808 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:A 2024-11-10T04:37:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:25,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:25,845 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/f7bd24dda3854068b680a09589e9922a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f7bd24dda3854068b680a09589e9922a 2024-11-10T04:37:25,850 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/c3cc2e4de6b24a76abde6bd2a7230691 2024-11-10T04:37:25,854 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/C of adb760d52bb12bfd79fa68b6162c27ba into f7bd24dda3854068b680a09589e9922a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:25,854 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:25,854 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/C, priority=10, startTime=1731213444932; duration=0sec 2024-11-10T04:37:25,854 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:25,854 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:C 2024-11-10T04:37:25,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/5d94ac9598f446f0849af7a60fc1fdd3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5d94ac9598f446f0849af7a60fc1fdd3 2024-11-10T04:37:25,864 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5d94ac9598f446f0849af7a60fc1fdd3, entries=150, sequenceid=165, filesize=30.4 K 2024-11-10T04:37:25,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/4136451a59f745398e0f11369e27453f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4136451a59f745398e0f11369e27453f 2024-11-10T04:37:25,871 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4136451a59f745398e0f11369e27453f, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T04:37:25,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/c3cc2e4de6b24a76abde6bd2a7230691 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c3cc2e4de6b24a76abde6bd2a7230691 2024-11-10T04:37:25,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213505872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:25,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213505873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:25,878 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c3cc2e4de6b24a76abde6bd2a7230691, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T04:37:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213505873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:25,879 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=174.43 KB/178620 for adb760d52bb12bfd79fa68b6162c27ba in 918ms, sequenceid=165, compaction requested=false 2024-11-10T04:37:25,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:25,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:25,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-10T04:37:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-10T04:37:25,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T04:37:25,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6870 sec 2024-11-10T04:37:25,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6920 sec 2024-11-10T04:37:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:25,983 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:25,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-10T04:37:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213505985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:25,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213505989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:25,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110810697ec88fa472393b513528ff94a82_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213445981/Put/seqid=0 2024-11-10T04:37:25,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213505993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:25,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:25,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741986_1162 (size=19774) 2024-11-10T04:37:26,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110810697ec88fa472393b513528ff94a82_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110810697ec88fa472393b513528ff94a82_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:26,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/5b90b25ee38e4ff592361e0f94fe9713, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:26,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/5b90b25ee38e4ff592361e0f94fe9713 is 175, key is test_row_0/A:col10/1731213445981/Put/seqid=0 2024-11-10T04:37:26,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741987_1163 (size=57033) 2024-11-10T04:37:26,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213506091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213506101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213506101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213506301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213506311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213506311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T04:37:26,342 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:26,342 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-10T04:37:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T04:37:26,344 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:26,345 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:26,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:26,418 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/5b90b25ee38e4ff592361e0f94fe9713 2024-11-10T04:37:26,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/2f9b4eda059049719c112605c7dd1ea6 is 50, key is test_row_0/B:col10/1731213445981/Put/seqid=0 2024-11-10T04:37:26,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741988_1164 (size=12151) 2024-11-10T04:37:26,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/2f9b4eda059049719c112605c7dd1ea6 2024-11-10T04:37:26,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/8244d63ca0474537970b8b98f76882e9 is 50, key is test_row_0/C:col10/1731213445981/Put/seqid=0 2024-11-10T04:37:26,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741989_1165 (size=12151) 2024-11-10T04:37:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T04:37:26,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T04:37:26,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:26,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:26,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213506611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213506621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213506621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T04:37:26,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:26,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:26,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T04:37:26,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213506691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213506691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:26,691 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,692 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:26,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T04:37:26,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:26,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:26,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:26,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/8244d63ca0474537970b8b98f76882e9 2024-11-10T04:37:26,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/5b90b25ee38e4ff592361e0f94fe9713 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5b90b25ee38e4ff592361e0f94fe9713 2024-11-10T04:37:26,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5b90b25ee38e4ff592361e0f94fe9713, entries=300, sequenceid=200, filesize=55.7 K 2024-11-10T04:37:26,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/2f9b4eda059049719c112605c7dd1ea6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2f9b4eda059049719c112605c7dd1ea6 2024-11-10T04:37:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2f9b4eda059049719c112605c7dd1ea6, entries=150, sequenceid=200, filesize=11.9 K 2024-11-10T04:37:26,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/8244d63ca0474537970b8b98f76882e9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8244d63ca0474537970b8b98f76882e9 2024-11-10T04:37:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8244d63ca0474537970b8b98f76882e9, entries=150, sequenceid=200, filesize=11.9 K 2024-11-10T04:37:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for adb760d52bb12bfd79fa68b6162c27ba in 887ms, sequenceid=200, compaction requested=true 2024-11-10T04:37:26,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:26,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:26,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:26,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:26,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T04:37:26,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:26,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-10T04:37:26,871 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,871 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,875 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,875 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/A is initiating minor compaction (all files) 2024-11-10T04:37:26,875 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/A in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,875 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/2c9365fcb62f4a6c9342c3be299dfe46, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5d94ac9598f446f0849af7a60fc1fdd3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5b90b25ee38e4ff592361e0f94fe9713] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=116.7 K 2024-11-10T04:37:26,875 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,875 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/2c9365fcb62f4a6c9342c3be299dfe46, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5d94ac9598f446f0849af7a60fc1fdd3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5b90b25ee38e4ff592361e0f94fe9713] 2024-11-10T04:37:26,876 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:26,876 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/C is initiating minor compaction (all files) 2024-11-10T04:37:26,876 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/C in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,876 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f7bd24dda3854068b680a09589e9922a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c3cc2e4de6b24a76abde6bd2a7230691, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8244d63ca0474537970b8b98f76882e9] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=35.9 K 2024-11-10T04:37:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,876 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c9365fcb62f4a6c9342c3be299dfe46, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731213442489 2024-11-10T04:37:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,876 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f7bd24dda3854068b680a09589e9922a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731213442489 2024-11-10T04:37:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,877 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c3cc2e4de6b24a76abde6bd2a7230691, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731213443647 2024-11-10T04:37:26,877 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d94ac9598f446f0849af7a60fc1fdd3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731213443647 2024-11-10T04:37:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,877 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8244d63ca0474537970b8b98f76882e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731213445869 2024-11-10T04:37:26,877 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b90b25ee38e4ff592361e0f94fe9713, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731213445861 2024-11-10T04:37:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,893 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,897 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#C#compaction#139 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,898 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/4117577bc84541c880dd62cfdb4b34c6 is 50, key is test_row_0/C:col10/1731213445981/Put/seqid=0 2024-11-10T04:37:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,899 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110463d405acef34f52882ce2aa211a3ee3_adb760d52bb12bfd79fa68b6162c27ba store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741990_1166 (size=12561) 2024-11-10T04:37:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,906 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110463d405acef34f52882ce2aa211a3ee3_adb760d52bb12bfd79fa68b6162c27ba, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:26,906 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110463d405acef34f52882ce2aa211a3ee3_adb760d52bb12bfd79fa68b6162c27ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,914 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/4117577bc84541c880dd62cfdb4b34c6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4117577bc84541c880dd62cfdb4b34c6 2024-11-10T04:37:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,921 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/C of adb760d52bb12bfd79fa68b6162c27ba into 4117577bc84541c880dd62cfdb4b34c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:26,921 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,921 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/C, priority=13, startTime=1731213446871; duration=0sec 2024-11-10T04:37:26,921 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:26,921 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:C 2024-11-10T04:37:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,922 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,923 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:26,923 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/B is initiating minor compaction (all files) 2024-11-10T04:37:26,923 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/B in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,923 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/b19625148d1c40788e11c700756b16a6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4136451a59f745398e0f11369e27453f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2f9b4eda059049719c112605c7dd1ea6] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=35.9 K 2024-11-10T04:37:26,924 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b19625148d1c40788e11c700756b16a6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731213442489 2024-11-10T04:37:26,924 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4136451a59f745398e0f11369e27453f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731213443647 2024-11-10T04:37:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,924 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9b4eda059049719c112605c7dd1ea6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731213445869 2024-11-10T04:37:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741991_1167 (size=4469) 2024-11-10T04:37:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,932 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#A#compaction#138 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,933 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9c4e913b524740aea7fa988cd93bbb6f is 175, key is test_row_0/A:col10/1731213445981/Put/seqid=0 2024-11-10T04:37:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,944 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#B#compaction#140 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:26,945 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/2fd65f7c88f14d0baebdf33bbd724035 is 50, key is test_row_0/B:col10/1731213445981/Put/seqid=0 2024-11-10T04:37:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741992_1168 (size=31515) 2024-11-10T04:37:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T04:37:26,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:26,959 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-10T04:37:26,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741993_1169 (size=12561) 2024-11-10T04:37:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,967 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/2fd65f7c88f14d0baebdf33bbd724035 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2fd65f7c88f14d0baebdf33bbd724035 2024-11-10T04:37:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T04:37:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,974 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/B of adb760d52bb12bfd79fa68b6162c27ba into 2fd65f7c88f14d0baebdf33bbd724035(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:26,974 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:26,975 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/B, priority=13, startTime=1731213446871; duration=0sec 2024-11-10T04:37:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,975 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:26,975 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:B 2024-11-10T04:37:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ab069c9eaf82412ea6f460447d45f046_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_1/A:col10/1731213445988/Put/seqid=0 2024-11-10T04:37:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741994_1170 (size=9814) 2024-11-10T04:37:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,996 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ab069c9eaf82412ea6f460447d45f046_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ab069c9eaf82412ea6f460447d45f046_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/12d0b60cd83d4ba7bf9099940d8621d2, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:26,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/12d0b60cd83d4ba7bf9099940d8621d2 is 175, key is test_row_1/A:col10/1731213445988/Put/seqid=0 2024-11-10T04:37:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741995_1171 (size=22461) 2024-11-10T04:37:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,011 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/12d0b60cd83d4ba7bf9099940d8621d2 2024-11-10T04:37:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/903e31ba32344b53b443525d955e7078 is 50, key is test_row_1/B:col10/1731213445988/Put/seqid=0 2024-11-10T04:37:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741996_1172 (size=9757) 2024-11-10T04:37:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,044 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/903e31ba32344b53b443525d955e7078 2024-11-10T04:37:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/4eab94ee38f141f28cfa694af69054ce is 50, key is test_row_1/C:col10/1731213445988/Put/seqid=0 2024-11-10T04:37:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741997_1173 (size=9757) 2024-11-10T04:37:27,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213507199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213507200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213507205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213507311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213507311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213507311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,358 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/9c4e913b524740aea7fa988cd93bbb6f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9c4e913b524740aea7fa988cd93bbb6f 2024-11-10T04:37:27,364 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/A of adb760d52bb12bfd79fa68b6162c27ba into 9c4e913b524740aea7fa988cd93bbb6f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:27,364 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:27,364 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/A, priority=13, startTime=1731213446871; duration=0sec 2024-11-10T04:37:27,364 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:27,365 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:A 2024-11-10T04:37:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T04:37:27,498 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/4eab94ee38f141f28cfa694af69054ce 2024-11-10T04:37:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/12d0b60cd83d4ba7bf9099940d8621d2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/12d0b60cd83d4ba7bf9099940d8621d2 2024-11-10T04:37:27,508 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/12d0b60cd83d4ba7bf9099940d8621d2, entries=100, sequenceid=206, filesize=21.9 K 2024-11-10T04:37:27,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/903e31ba32344b53b443525d955e7078 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/903e31ba32344b53b443525d955e7078 2024-11-10T04:37:27,514 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/903e31ba32344b53b443525d955e7078, entries=100, sequenceid=206, filesize=9.5 K 2024-11-10T04:37:27,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/4eab94ee38f141f28cfa694af69054ce as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4eab94ee38f141f28cfa694af69054ce 2024-11-10T04:37:27,519 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4eab94ee38f141f28cfa694af69054ce, entries=100, sequenceid=206, filesize=9.5 K 2024-11-10T04:37:27,520 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=194.56 KB/199230 for adb760d52bb12bfd79fa68b6162c27ba in 562ms, sequenceid=206, compaction requested=false 2024-11-10T04:37:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:27,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-10T04:37:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-10T04:37:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:27,524 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:27,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-10T04:37:27,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213507521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T04:37:27,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1770 sec 2024-11-10T04:37:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213507525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.1840 sec 2024-11-10T04:37:27,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213507526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101f5f9809009e4578a703debbac2664b8_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213447197/Put/seqid=0 2024-11-10T04:37:27,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741998_1174 (size=14794) 2024-11-10T04:37:27,550 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:27,555 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101f5f9809009e4578a703debbac2664b8_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101f5f9809009e4578a703debbac2664b8_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:27,556 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/91ae7b9919644d0b8eab9eab7031c675, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:27,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/91ae7b9919644d0b8eab9eab7031c675 is 175, key is test_row_0/A:col10/1731213447197/Put/seqid=0 2024-11-10T04:37:27,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741999_1175 (size=39749) 2024-11-10T04:37:27,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213507631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213507631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213507831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213507841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213507841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:27,963 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=69.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/91ae7b9919644d0b8eab9eab7031c675 2024-11-10T04:37:27,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/93b54f88e6c14f4b9d04442df1997464 is 50, key is test_row_0/B:col10/1731213447197/Put/seqid=0 2024-11-10T04:37:28,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742000_1176 (size=12151) 2024-11-10T04:37:28,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/93b54f88e6c14f4b9d04442df1997464 2024-11-10T04:37:28,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/fc7ba1e08ba04208a33ad06125757535 is 50, key is test_row_0/C:col10/1731213447197/Put/seqid=0 2024-11-10T04:37:28,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742001_1177 (size=12151) 2024-11-10T04:37:28,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:28,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213508151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213508151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:28,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213508341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:28,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/fc7ba1e08ba04208a33ad06125757535 2024-11-10T04:37:28,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/91ae7b9919644d0b8eab9eab7031c675 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/91ae7b9919644d0b8eab9eab7031c675 2024-11-10T04:37:28,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/91ae7b9919644d0b8eab9eab7031c675, entries=200, sequenceid=242, filesize=38.8 K 2024-11-10T04:37:28,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/93b54f88e6c14f4b9d04442df1997464 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/93b54f88e6c14f4b9d04442df1997464 2024-11-10T04:37:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/93b54f88e6c14f4b9d04442df1997464, entries=150, sequenceid=242, filesize=11.9 K 2024-11-10T04:37:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/fc7ba1e08ba04208a33ad06125757535 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/fc7ba1e08ba04208a33ad06125757535 2024-11-10T04:37:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/fc7ba1e08ba04208a33ad06125757535, entries=150, sequenceid=242, filesize=11.9 K 2024-11-10T04:37:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~207.98 KB/212970, heapSize ~545.63 KB/558720, currentSize=0 B/0 for adb760d52bb12bfd79fa68b6162c27ba in 928ms, sequenceid=242, compaction requested=true 2024-11-10T04:37:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:28,453 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:28,453 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,454 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:28,454 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:28,454 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/A is initiating minor compaction (all files) 2024-11-10T04:37:28,454 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/B is initiating minor compaction (all files) 2024-11-10T04:37:28,454 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/A in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:28,454 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/B in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,454 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2fd65f7c88f14d0baebdf33bbd724035, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/903e31ba32344b53b443525d955e7078, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/93b54f88e6c14f4b9d04442df1997464] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=33.7 K 2024-11-10T04:37:28,454 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9c4e913b524740aea7fa988cd93bbb6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/12d0b60cd83d4ba7bf9099940d8621d2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/91ae7b9919644d0b8eab9eab7031c675] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=91.5 K 2024-11-10T04:37:28,455 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:28,455 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9c4e913b524740aea7fa988cd93bbb6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/12d0b60cd83d4ba7bf9099940d8621d2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/91ae7b9919644d0b8eab9eab7031c675] 2024-11-10T04:37:28,455 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c4e913b524740aea7fa988cd93bbb6f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731213445869 2024-11-10T04:37:28,455 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fd65f7c88f14d0baebdf33bbd724035, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731213445869 2024-11-10T04:37:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,456 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 903e31ba32344b53b443525d955e7078, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731213445987 2024-11-10T04:37:28,456 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d0b60cd83d4ba7bf9099940d8621d2, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731213445987 2024-11-10T04:37:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,456 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 93b54f88e6c14f4b9d04442df1997464, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731213447197 2024-11-10T04:37:28,456 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91ae7b9919644d0b8eab9eab7031c675, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731213447197 2024-11-10T04:37:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,465 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,467 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,468 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/fdc4a97887ac4a23866fa2cb461cca65 is 50, key is test_row_0/B:col10/1731213447197/Put/seqid=0 2024-11-10T04:37:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,468 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110950909555b8d4741b32f4215be59ef56_adb760d52bb12bfd79fa68b6162c27ba store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,470 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110950909555b8d4741b32f4215be59ef56_adb760d52bb12bfd79fa68b6162c27ba, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:28,470 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110950909555b8d4741b32f4215be59ef56_adb760d52bb12bfd79fa68b6162c27ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T04:37:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,491 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,493 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742003_1179 (size=4469) 2024-11-10T04:37:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-10T04:37:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T04:37:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742002_1178 (size=12663) 2024-11-10T04:37:28,496 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,497 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:28,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T04:37:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T04:37:28,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-10T04:37:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-10T04:37:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-10T04:37:28,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-10T04:37:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 164 msec 2024-11-10T04:37:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:28,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:37:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103bac530d0a2542d6af1e1540280a53f0_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213448730/Put/seqid=0 2024-11-10T04:37:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742005_1181 (size=24758) 2024-11-10T04:37:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,750 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,756 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103bac530d0a2542d6af1e1540280a53f0_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103bac530d0a2542d6af1e1540280a53f0_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:28,758 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/f990f984d93a4ebdba311e42b36f344a, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/f990f984d93a4ebdba311e42b36f344a is 175, key is test_row_0/A:col10/1731213448730/Put/seqid=0 2024-11-10T04:37:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742004_1180 (size=74395) 2024-11-10T04:37:28,762 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/f990f984d93a4ebdba311e42b36f344a 2024-11-10T04:37:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:28,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/e99f80e7fcf4446c9cbafef926ee0804 is 50, key is test_row_0/B:col10/1731213448730/Put/seqid=0 2024-11-10T04:37:28,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742006_1182 (size=12151) 2024-11-10T04:37:28,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/e99f80e7fcf4446c9cbafef926ee0804 2024-11-10T04:37:28,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/6138507c03674dc5978e420e8d57a8b4 is 50, key is test_row_0/C:col10/1731213448730/Put/seqid=0 2024-11-10T04:37:28,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742007_1183 (size=12151) 2024-11-10T04:37:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T04:37:28,812 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:28,812 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-10T04:37:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T04:37:28,815 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:28,815 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:28,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:28,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213508813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213508813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:28,896 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#A#compaction#147 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:28,897 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/a18aa2f17c1f4cecbb6b18210ff2682e is 175, key is test_row_0/A:col10/1731213447197/Put/seqid=0 2024-11-10T04:37:28,903 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/fdc4a97887ac4a23866fa2cb461cca65 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/fdc4a97887ac4a23866fa2cb461cca65 2024-11-10T04:37:28,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742008_1184 (size=31617) 2024-11-10T04:37:28,910 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/B of adb760d52bb12bfd79fa68b6162c27ba into fdc4a97887ac4a23866fa2cb461cca65(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:28,910 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:28,910 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/B, priority=13, startTime=1731213448453; duration=0sec 2024-11-10T04:37:28,910 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:28,911 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:B 2024-11-10T04:37:28,911 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:28,913 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:28,913 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/C is initiating minor compaction (all files) 2024-11-10T04:37:28,913 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/C in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:28,913 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4117577bc84541c880dd62cfdb4b34c6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4eab94ee38f141f28cfa694af69054ce, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/fc7ba1e08ba04208a33ad06125757535] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=33.7 K 2024-11-10T04:37:28,914 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4117577bc84541c880dd62cfdb4b34c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731213445869 2024-11-10T04:37:28,914 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/a18aa2f17c1f4cecbb6b18210ff2682e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/a18aa2f17c1f4cecbb6b18210ff2682e 2024-11-10T04:37:28,914 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eab94ee38f141f28cfa694af69054ce, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731213445987 2024-11-10T04:37:28,915 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7ba1e08ba04208a33ad06125757535, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731213447197 2024-11-10T04:37:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T04:37:28,921 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/A of adb760d52bb12bfd79fa68b6162c27ba into a18aa2f17c1f4cecbb6b18210ff2682e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:28,921 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:28,921 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/A, priority=13, startTime=1731213448453; duration=0sec 2024-11-10T04:37:28,921 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:28,921 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:A 2024-11-10T04:37:28,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213508921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213508921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:28,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:28,927 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#C#compaction#152 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:28,927 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/3cf4474397f5461bb9fe0307bb635f1e is 50, key is test_row_0/C:col10/1731213447197/Put/seqid=0 2024-11-10T04:37:28,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742009_1185 (size=12663) 2024-11-10T04:37:28,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T04:37:28,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:28,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:28,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:28,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:29,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T04:37:29,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:29,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:29,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:29,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:29,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T04:37:29,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213509132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:29,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213509132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:29,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/6138507c03674dc5978e420e8d57a8b4 2024-11-10T04:37:29,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/f990f984d93a4ebdba311e42b36f344a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/f990f984d93a4ebdba311e42b36f344a 2024-11-10T04:37:29,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/f990f984d93a4ebdba311e42b36f344a, entries=400, sequenceid=254, filesize=72.7 K 2024-11-10T04:37:29,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/e99f80e7fcf4446c9cbafef926ee0804 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e99f80e7fcf4446c9cbafef926ee0804 2024-11-10T04:37:29,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e99f80e7fcf4446c9cbafef926ee0804, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T04:37:29,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/6138507c03674dc5978e420e8d57a8b4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6138507c03674dc5978e420e8d57a8b4 2024-11-10T04:37:29,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6138507c03674dc5978e420e8d57a8b4, entries=150, sequenceid=254, filesize=11.9 K 2024-11-10T04:37:29,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for adb760d52bb12bfd79fa68b6162c27ba in 503ms, sequenceid=254, compaction requested=false 2024-11-10T04:37:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:29,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T04:37:29,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:29,275 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:37:29,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110527a9d7fb7d142d883c3d5718427fd94_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213448804/Put/seqid=0 2024-11-10T04:37:29,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742010_1186 (size=12454) 2024-11-10T04:37:29,340 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/3cf4474397f5461bb9fe0307bb635f1e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/3cf4474397f5461bb9fe0307bb635f1e 2024-11-10T04:37:29,346 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/C of adb760d52bb12bfd79fa68b6162c27ba into 3cf4474397f5461bb9fe0307bb635f1e(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:29,346 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:29,346 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/C, priority=13, startTime=1731213448453; duration=0sec 2024-11-10T04:37:29,346 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:29,346 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:C 2024-11-10T04:37:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:29,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:29,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213509384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T04:37:29,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213509442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:29,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213509442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:29,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213509491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:29,694 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110527a9d7fb7d142d883c3d5718427fd94_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110527a9d7fb7d142d883c3d5718427fd94_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/e2908e4ec95c4d4d87511f0d02082da8, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:29,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/e2908e4ec95c4d4d87511f0d02082da8 is 175, key is test_row_0/A:col10/1731213448804/Put/seqid=0 2024-11-10T04:37:29,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213509701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:29,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742011_1187 (size=31255) 2024-11-10T04:37:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T04:37:29,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213509951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:29,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213509951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:30,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213510011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:30,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:30,106 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/e2908e4ec95c4d4d87511f0d02082da8 2024-11-10T04:37:30,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/9e8b1061a84a472d997f9d8327b5a35c is 50, key is test_row_0/B:col10/1731213448804/Put/seqid=0 2024-11-10T04:37:30,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742012_1188 (size=12301) 2024-11-10T04:37:30,124 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/9e8b1061a84a472d997f9d8327b5a35c 2024-11-10T04:37:30,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/66eec9894e2c4a16af8873a1562596db is 50, key is test_row_0/C:col10/1731213448804/Put/seqid=0 2024-11-10T04:37:30,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742013_1189 (size=12301) 2024-11-10T04:37:30,306 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:37:30,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213510521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:30,538 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/66eec9894e2c4a16af8873a1562596db 2024-11-10T04:37:30,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/e2908e4ec95c4d4d87511f0d02082da8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e2908e4ec95c4d4d87511f0d02082da8 2024-11-10T04:37:30,550 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e2908e4ec95c4d4d87511f0d02082da8, entries=150, sequenceid=282, filesize=30.5 K 2024-11-10T04:37:30,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/9e8b1061a84a472d997f9d8327b5a35c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/9e8b1061a84a472d997f9d8327b5a35c 2024-11-10T04:37:30,556 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/9e8b1061a84a472d997f9d8327b5a35c, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T04:37:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/66eec9894e2c4a16af8873a1562596db as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/66eec9894e2c4a16af8873a1562596db 2024-11-10T04:37:30,562 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/66eec9894e2c4a16af8873a1562596db, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T04:37:30,563 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for adb760d52bb12bfd79fa68b6162c27ba in 1288ms, sequenceid=282, compaction requested=true 2024-11-10T04:37:30,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:30,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:30,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-10T04:37:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-10T04:37:30,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-10T04:37:30,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7500 sec 2024-11-10T04:37:30,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.7550 sec 2024-11-10T04:37:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:30,725 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:30,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:37:30,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110167ae4c85c9b4d61b02348500dc6d5ae_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213449380/Put/seqid=0 2024-11-10T04:37:30,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742014_1190 (size=17534) 2024-11-10T04:37:30,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213510779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:30,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213510786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:30,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213510891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213510891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T04:37:30,962 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:30,962 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-10T04:37:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T04:37:30,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:30,965 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:30,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:30,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213510962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:30,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213510962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:30,967 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,967 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T04:37:31,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213511101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:31,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213511101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:31,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,147 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:31,151 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110167ae4c85c9b4d61b02348500dc6d5ae_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110167ae4c85c9b4d61b02348500dc6d5ae_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:31,152 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/7245b491668545949ff6db8fa3edd7e5, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:31,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/7245b491668545949ff6db8fa3edd7e5 is 175, key is test_row_0/A:col10/1731213449380/Put/seqid=0 2024-11-10T04:37:31,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742015_1191 (size=48639) 2024-11-10T04:37:31,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:31,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:31,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T04:37:31,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213511411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:31,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213511411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:31,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213511531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:31,538 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:31,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:31,557 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/7245b491668545949ff6db8fa3edd7e5 2024-11-10T04:37:31,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/cc72dbf99a0047f4a8d18ad5594e8c05 is 50, key is test_row_0/B:col10/1731213449380/Put/seqid=0 2024-11-10T04:37:31,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742016_1192 (size=12301) 2024-11-10T04:37:31,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T04:37:31,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:31,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:31,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:31,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:31,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213511921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:31,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213511931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:31,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:31,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/cc72dbf99a0047f4a8d18ad5594e8c05 2024-11-10T04:37:31,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/eaddccdd21c84e51864280d7dad85ad0 is 50, key is test_row_0/C:col10/1731213449380/Put/seqid=0 2024-11-10T04:37:31,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742017_1193 (size=12301) 2024-11-10T04:37:32,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:32,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:32,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:32,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T04:37:32,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:32,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:32,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:32,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:32,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:32,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/eaddccdd21c84e51864280d7dad85ad0 2024-11-10T04:37:32,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/7245b491668545949ff6db8fa3edd7e5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7245b491668545949ff6db8fa3edd7e5 2024-11-10T04:37:32,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7245b491668545949ff6db8fa3edd7e5, entries=250, sequenceid=295, filesize=47.5 K 2024-11-10T04:37:32,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/cc72dbf99a0047f4a8d18ad5594e8c05 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cc72dbf99a0047f4a8d18ad5594e8c05 2024-11-10T04:37:32,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cc72dbf99a0047f4a8d18ad5594e8c05, entries=150, sequenceid=295, filesize=12.0 K 2024-11-10T04:37:32,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/eaddccdd21c84e51864280d7dad85ad0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eaddccdd21c84e51864280d7dad85ad0 2024-11-10T04:37:32,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eaddccdd21c84e51864280d7dad85ad0, entries=150, sequenceid=295, filesize=12.0 K 2024-11-10T04:37:32,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for adb760d52bb12bfd79fa68b6162c27ba in 1680ms, sequenceid=295, compaction requested=true 2024-11-10T04:37:32,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:32,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:32,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:32,405 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:32,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:32,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:32,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:32,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:32,405 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:32,407 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:32,407 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 185906 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:32,407 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/B is initiating minor compaction (all files) 2024-11-10T04:37:32,407 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/A is initiating minor compaction (all files) 2024-11-10T04:37:32,407 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/A in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,407 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/B in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,407 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/a18aa2f17c1f4cecbb6b18210ff2682e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/f990f984d93a4ebdba311e42b36f344a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e2908e4ec95c4d4d87511f0d02082da8, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7245b491668545949ff6db8fa3edd7e5] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=181.5 K 2024-11-10T04:37:32,407 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/fdc4a97887ac4a23866fa2cb461cca65, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e99f80e7fcf4446c9cbafef926ee0804, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/9e8b1061a84a472d997f9d8327b5a35c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cc72dbf99a0047f4a8d18ad5594e8c05] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=48.3 K 2024-11-10T04:37:32,407 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,407 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/a18aa2f17c1f4cecbb6b18210ff2682e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/f990f984d93a4ebdba311e42b36f344a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e2908e4ec95c4d4d87511f0d02082da8, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7245b491668545949ff6db8fa3edd7e5] 2024-11-10T04:37:32,408 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc4a97887ac4a23866fa2cb461cca65, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731213447197 2024-11-10T04:37:32,408 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a18aa2f17c1f4cecbb6b18210ff2682e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731213447197 2024-11-10T04:37:32,408 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f990f984d93a4ebdba311e42b36f344a, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731213448710 2024-11-10T04:37:32,408 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e99f80e7fcf4446c9cbafef926ee0804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731213448712 2024-11-10T04:37:32,409 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2908e4ec95c4d4d87511f0d02082da8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731213448804 2024-11-10T04:37:32,409 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e8b1061a84a472d997f9d8327b5a35c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731213448804 2024-11-10T04:37:32,409 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cc72dbf99a0047f4a8d18ad5594e8c05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731213449380 2024-11-10T04:37:32,409 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7245b491668545949ff6db8fa3edd7e5, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731213449378 2024-11-10T04:37:32,419 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:32,420 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:32,421 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/00dceeb7afcd4962b760fe600424301b is 50, key is test_row_0/B:col10/1731213449380/Put/seqid=0 2024-11-10T04:37:32,424 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110772b6bb896544bb686dcdc974dcaf10c_adb760d52bb12bfd79fa68b6162c27ba store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:32,427 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110772b6bb896544bb686dcdc974dcaf10c_adb760d52bb12bfd79fa68b6162c27ba, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:32,427 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110772b6bb896544bb686dcdc974dcaf10c_adb760d52bb12bfd79fa68b6162c27ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:32,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742018_1194 (size=12949) 2024-11-10T04:37:32,456 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/00dceeb7afcd4962b760fe600424301b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/00dceeb7afcd4962b760fe600424301b 2024-11-10T04:37:32,463 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/B of adb760d52bb12bfd79fa68b6162c27ba into 00dceeb7afcd4962b760fe600424301b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:32,463 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:32,463 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/B, priority=12, startTime=1731213452405; duration=0sec 2024-11-10T04:37:32,463 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:32,464 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:B 2024-11-10T04:37:32,464 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:32,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742019_1195 (size=4469) 2024-11-10T04:37:32,468 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:32,468 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/C is initiating minor compaction (all files) 2024-11-10T04:37:32,468 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#A#compaction#160 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:32,468 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/C in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,469 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/3cf4474397f5461bb9fe0307bb635f1e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6138507c03674dc5978e420e8d57a8b4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/66eec9894e2c4a16af8873a1562596db, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eaddccdd21c84e51864280d7dad85ad0] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=48.3 K 2024-11-10T04:37:32,469 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/cd88871f23c6461ebe5aea233b7c20e7 is 175, key is test_row_0/A:col10/1731213449380/Put/seqid=0 2024-11-10T04:37:32,469 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cf4474397f5461bb9fe0307bb635f1e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731213447197 2024-11-10T04:37:32,471 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6138507c03674dc5978e420e8d57a8b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731213448712 2024-11-10T04:37:32,472 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 66eec9894e2c4a16af8873a1562596db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731213448804 2024-11-10T04:37:32,472 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting eaddccdd21c84e51864280d7dad85ad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731213449380 2024-11-10T04:37:32,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742020_1196 (size=31903) 2024-11-10T04:37:32,491 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:32,491 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/d04556ee15d0403dbdbe49133954c372 is 50, key is test_row_0/C:col10/1731213449380/Put/seqid=0 2024-11-10T04:37:32,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T04:37:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:32,494 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T04:37:32,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742021_1197 (size=12949) 2024-11-10T04:37:32,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e8961f6e92c1440dab2a20814c63024e_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213450761/Put/seqid=0 2024-11-10T04:37:32,505 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/d04556ee15d0403dbdbe49133954c372 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d04556ee15d0403dbdbe49133954c372 2024-11-10T04:37:32,511 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/C of adb760d52bb12bfd79fa68b6162c27ba into d04556ee15d0403dbdbe49133954c372(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:32,511 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:32,511 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/C, priority=12, startTime=1731213452405; duration=0sec 2024-11-10T04:37:32,511 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:32,511 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:C 2024-11-10T04:37:32,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742022_1198 (size=12454) 2024-11-10T04:37:32,892 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/cd88871f23c6461ebe5aea233b7c20e7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/cd88871f23c6461ebe5aea233b7c20e7 2024-11-10T04:37:32,899 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/A of adb760d52bb12bfd79fa68b6162c27ba into cd88871f23c6461ebe5aea233b7c20e7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:32,899 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:32,899 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/A, priority=12, startTime=1731213452405; duration=0sec 2024-11-10T04:37:32,900 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:32,900 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:A 2024-11-10T04:37:32,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:32,918 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e8961f6e92c1440dab2a20814c63024e_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e8961f6e92c1440dab2a20814c63024e_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/b6ade4985f8d458caf5421bef82752ae, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:32,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/b6ade4985f8d458caf5421bef82752ae is 175, key is test_row_0/A:col10/1731213450761/Put/seqid=0 2024-11-10T04:37:32,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742023_1199 (size=31255) 2024-11-10T04:37:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:32,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. as already flushing 2024-11-10T04:37:32,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213512953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:32,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:32,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:32,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213512954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:32,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:32,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:32,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:32,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:32,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34506 deadline: 1731213512971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:32,975 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:32,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34528 deadline: 1731213512991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:32,994 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:33,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213513061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:33,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213513061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T04:37:33,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213513271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:33,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213513271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:33,325 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/b6ade4985f8d458caf5421bef82752ae 2024-11-10T04:37:33,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/d7af763106964d428e64e2dbcf9f67f1 is 50, key is test_row_0/B:col10/1731213450761/Put/seqid=0 2024-11-10T04:37:33,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742024_1200 (size=12301) 2024-11-10T04:37:33,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34530 deadline: 1731213513561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:33,562 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in adb760d52bb12bfd79fa68b6162c27ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:33,579 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-10T04:37:33,579 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:33,579 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:33,579 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:33,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:33,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213513581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:33,581 INFO [Thread-619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-619. 2024-11-10T04:37:33,582 DEBUG [Thread-619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:33,582 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:33,582 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:33,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:33,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:33,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213513581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:33,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:33,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:33,583 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-11-10T04:37:33,583 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:33,583 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:33,583 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:33,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:33,586 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-11-10T04:37:33,586 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:33,586 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:33,586 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:33,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:33,746 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/d7af763106964d428e64e2dbcf9f67f1 2024-11-10T04:37:33,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/581d6cb4030d420d9161afd5987457ac is 50, key is test_row_0/C:col10/1731213450761/Put/seqid=0 2024-11-10T04:37:33,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742025_1201 (size=12301) 2024-11-10T04:37:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34532 deadline: 1731213514092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34538 deadline: 1731213514091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:37:34,157 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/581d6cb4030d420d9161afd5987457ac 2024-11-10T04:37:34,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/b6ade4985f8d458caf5421bef82752ae as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/b6ade4985f8d458caf5421bef82752ae 2024-11-10T04:37:34,166 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/b6ade4985f8d458caf5421bef82752ae, entries=150, sequenceid=320, filesize=30.5 K 2024-11-10T04:37:34,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/d7af763106964d428e64e2dbcf9f67f1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7af763106964d428e64e2dbcf9f67f1 2024-11-10T04:37:34,170 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7af763106964d428e64e2dbcf9f67f1, entries=150, sequenceid=320, filesize=12.0 K 2024-11-10T04:37:34,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/581d6cb4030d420d9161afd5987457ac as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/581d6cb4030d420d9161afd5987457ac 2024-11-10T04:37:34,175 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/581d6cb4030d420d9161afd5987457ac, entries=150, sequenceid=320, filesize=12.0 K 2024-11-10T04:37:34,176 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for adb760d52bb12bfd79fa68b6162c27ba in 1681ms, sequenceid=320, compaction requested=false 2024-11-10T04:37:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-10T04:37:34,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-10T04:37:34,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T04:37:34,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2120 sec 2024-11-10T04:37:34,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 3.2170 sec 2024-11-10T04:37:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:35,102 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:35,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T04:37:35,103 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-11-10T04:37:35,103 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-11-10T04:37:35,103 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:35,103 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:35,103 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:35,103 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:35,103 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:35,103 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:35,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:35,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:35,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106a21befeb6e24b338907064f0c0d6fad_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_0/A:col10/1731213452941/Put/seqid=0 2024-11-10T04:37:35,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742026_1202 (size=12454) 2024-11-10T04:37:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T04:37:35,132 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:35,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:35,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106a21befeb6e24b338907064f0c0d6fad_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a21befeb6e24b338907064f0c0d6fad_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:35,519 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/ad70979812584c8589e6ff736097b549, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:35,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/ad70979812584c8589e6ff736097b549 is 175, key is test_row_0/A:col10/1731213452941/Put/seqid=0 2024-11-10T04:37:35,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742027_1203 (size=31255) 2024-11-10T04:37:35,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/ad70979812584c8589e6ff736097b549 2024-11-10T04:37:35,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/5f81934a112e4d16bf1289a00ff52a93 is 50, key is test_row_0/B:col10/1731213452941/Put/seqid=0 2024-11-10T04:37:35,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742028_1204 (size=12301) 2024-11-10T04:37:36,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/5f81934a112e4d16bf1289a00ff52a93 2024-11-10T04:37:36,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/c078ce6c5b824ba8b1d66a76b01ea32f is 50, key is test_row_0/C:col10/1731213452941/Put/seqid=0 2024-11-10T04:37:36,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742029_1205 (size=12301) 2024-11-10T04:37:36,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/c078ce6c5b824ba8b1d66a76b01ea32f 2024-11-10T04:37:36,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/ad70979812584c8589e6ff736097b549 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/ad70979812584c8589e6ff736097b549 2024-11-10T04:37:36,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/ad70979812584c8589e6ff736097b549, entries=150, sequenceid=337, filesize=30.5 K 2024-11-10T04:37:36,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/5f81934a112e4d16bf1289a00ff52a93 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/5f81934a112e4d16bf1289a00ff52a93 2024-11-10T04:37:36,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/5f81934a112e4d16bf1289a00ff52a93, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T04:37:36,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/c078ce6c5b824ba8b1d66a76b01ea32f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c078ce6c5b824ba8b1d66a76b01ea32f 2024-11-10T04:37:36,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c078ce6c5b824ba8b1d66a76b01ea32f, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T04:37:36,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for adb760d52bb12bfd79fa68b6162c27ba in 1663ms, sequenceid=337, compaction requested=true 2024-11-10T04:37:36,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:36,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:36,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:36,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:36,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:36,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb760d52bb12bfd79fa68b6162c27ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:36,765 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:36,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:36,765 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:36,766 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:36,766 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:36,766 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/B is initiating minor compaction (all files) 2024-11-10T04:37:36,766 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/A is initiating minor compaction (all files) 2024-11-10T04:37:36,766 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/B in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:36,766 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/A in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:36,766 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/00dceeb7afcd4962b760fe600424301b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7af763106964d428e64e2dbcf9f67f1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/5f81934a112e4d16bf1289a00ff52a93] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=36.7 K 2024-11-10T04:37:36,766 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/cd88871f23c6461ebe5aea233b7c20e7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/b6ade4985f8d458caf5421bef82752ae, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/ad70979812584c8589e6ff736097b549] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=92.2 K 2024-11-10T04:37:36,766 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:36,766 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/cd88871f23c6461ebe5aea233b7c20e7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/b6ade4985f8d458caf5421bef82752ae, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/ad70979812584c8589e6ff736097b549] 2024-11-10T04:37:36,766 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 00dceeb7afcd4962b760fe600424301b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731213449380 2024-11-10T04:37:36,767 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d7af763106964d428e64e2dbcf9f67f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731213450761 2024-11-10T04:37:36,767 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd88871f23c6461ebe5aea233b7c20e7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731213449380 2024-11-10T04:37:36,767 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f81934a112e4d16bf1289a00ff52a93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731213452941 2024-11-10T04:37:36,767 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6ade4985f8d458caf5421bef82752ae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731213450761 2024-11-10T04:37:36,767 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad70979812584c8589e6ff736097b549, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731213452941 2024-11-10T04:37:36,774 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:36,775 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:36,776 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/b4c4184ca54b403e9b417fbc9ce0e6be is 50, key is test_row_0/B:col10/1731213452941/Put/seqid=0 2024-11-10T04:37:36,776 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110e98ed1fb12294e42bfa606d541261dcd_adb760d52bb12bfd79fa68b6162c27ba store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:36,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742030_1206 (size=13051) 2024-11-10T04:37:36,804 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110e98ed1fb12294e42bfa606d541261dcd_adb760d52bb12bfd79fa68b6162c27ba, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:36,804 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e98ed1fb12294e42bfa606d541261dcd_adb760d52bb12bfd79fa68b6162c27ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:36,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742031_1207 (size=4469) 2024-11-10T04:37:37,002 INFO [Thread-606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-606. 2024-11-10T04:37:37,003 DEBUG [Thread-606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:37,003 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:37,003 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:37,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:37,012 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-11-10T04:37:37,012 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:37,013 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:37,013 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:37,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:37,184 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/b4c4184ca54b403e9b417fbc9ce0e6be as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/b4c4184ca54b403e9b417fbc9ce0e6be 2024-11-10T04:37:37,189 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/B of adb760d52bb12bfd79fa68b6162c27ba into b4c4184ca54b403e9b417fbc9ce0e6be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:37,189 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:37,189 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/B, priority=13, startTime=1731213456765; duration=0sec 2024-11-10T04:37:37,189 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:37,189 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:B 2024-11-10T04:37:37,189 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:37,190 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:37,190 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): adb760d52bb12bfd79fa68b6162c27ba/C is initiating minor compaction (all files) 2024-11-10T04:37:37,190 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb760d52bb12bfd79fa68b6162c27ba/C in TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:37,190 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d04556ee15d0403dbdbe49133954c372, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/581d6cb4030d420d9161afd5987457ac, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c078ce6c5b824ba8b1d66a76b01ea32f] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp, totalSize=36.7 K 2024-11-10T04:37:37,190 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d04556ee15d0403dbdbe49133954c372, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731213449380 2024-11-10T04:37:37,191 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 581d6cb4030d420d9161afd5987457ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731213450761 2024-11-10T04:37:37,191 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c078ce6c5b824ba8b1d66a76b01ea32f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731213452941 2024-11-10T04:37:37,198 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:37,199 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/1ec6fe7cb5464390a3a4d60011069a4b is 50, key is test_row_0/C:col10/1731213452941/Put/seqid=0 2024-11-10T04:37:37,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742032_1208 (size=13051) 2024-11-10T04:37:37,209 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb760d52bb12bfd79fa68b6162c27ba#A#compaction#169 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:37,209 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/64ae4c5fb7d144be9c65e5f2da479286 is 175, key is test_row_0/A:col10/1731213452941/Put/seqid=0 2024-11-10T04:37:37,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742033_1209 (size=32005) 2024-11-10T04:37:37,572 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-10T04:37:37,572 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:37:37,572 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:37,572 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:37,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:37,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3583 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3415 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1515 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4545 rows 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1514 2024-11-10T04:37:37,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4539 rows 2024-11-10T04:37:37,573 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:37:37,573 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:37:37,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:37,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:37,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:37:37,574 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T04:37:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T04:37:37,577 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213457577"}]},"ts":"1731213457577"} 2024-11-10T04:37:37,578 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T04:37:37,578 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T04:37:37,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T04:37:37,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, UNASSIGN}] 2024-11-10T04:37:37,581 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, UNASSIGN 2024-11-10T04:37:37,581 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=adb760d52bb12bfd79fa68b6162c27ba, regionState=CLOSING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:37,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, UNASSIGN because future has completed 2024-11-10T04:37:37,583 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T04:37:37,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:37:37,607 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/1ec6fe7cb5464390a3a4d60011069a4b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/1ec6fe7cb5464390a3a4d60011069a4b 2024-11-10T04:37:37,611 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/C of adb760d52bb12bfd79fa68b6162c27ba into 1ec6fe7cb5464390a3a4d60011069a4b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:37,611 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:37,611 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/C, priority=13, startTime=1731213456765; duration=0sec 2024-11-10T04:37:37,611 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:37,611 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:C 2024-11-10T04:37:37,617 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/64ae4c5fb7d144be9c65e5f2da479286 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/64ae4c5fb7d144be9c65e5f2da479286 2024-11-10T04:37:37,621 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb760d52bb12bfd79fa68b6162c27ba/A of adb760d52bb12bfd79fa68b6162c27ba into 64ae4c5fb7d144be9c65e5f2da479286(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:37,621 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb760d52bb12bfd79fa68b6162c27ba: 2024-11-10T04:37:37,621 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., storeName=adb760d52bb12bfd79fa68b6162c27ba/A, priority=13, startTime=1731213456765; duration=0sec 2024-11-10T04:37:37,621 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:37,621 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb760d52bb12bfd79fa68b6162c27ba:A 2024-11-10T04:37:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T04:37:37,734 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:37,735 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T04:37:37,735 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing adb760d52bb12bfd79fa68b6162c27ba, disabling compactions & flushes 2024-11-10T04:37:37,735 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:37,735 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:37,735 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. after waiting 0 ms 2024-11-10T04:37:37,735 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:37,735 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing adb760d52bb12bfd79fa68b6162c27ba 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-10T04:37:37,740 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111094ea5c3b20f248bdb54e06927d562b18_adb760d52bb12bfd79fa68b6162c27ba is 50, key is test_row_2/A:col10/1731213457571/Put/seqid=0 2024-11-10T04:37:37,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742034_1210 (size=7374) 2024-11-10T04:37:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T04:37:38,144 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:38,148 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111094ea5c3b20f248bdb54e06927d562b18_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111094ea5c3b20f248bdb54e06927d562b18_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:38,149 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/e298baa8003d4841a3f3f43fa4fac40a, store: [table=TestAcidGuarantees family=A region=adb760d52bb12bfd79fa68b6162c27ba] 2024-11-10T04:37:38,149 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/e298baa8003d4841a3f3f43fa4fac40a is 175, key is test_row_2/A:col10/1731213457571/Put/seqid=0 2024-11-10T04:37:38,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742035_1211 (size=13865) 2024-11-10T04:37:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T04:37:38,554 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/e298baa8003d4841a3f3f43fa4fac40a 2024-11-10T04:37:38,561 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/cafaa87cf75b4004a2934c8cf26935bb is 50, key is test_row_2/B:col10/1731213457571/Put/seqid=0 2024-11-10T04:37:38,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742036_1212 (size=7415) 2024-11-10T04:37:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T04:37:38,965 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/cafaa87cf75b4004a2934c8cf26935bb 2024-11-10T04:37:38,972 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/d9f0b36ca44f467e8b99c3de7534c603 is 50, key is test_row_2/C:col10/1731213457571/Put/seqid=0 2024-11-10T04:37:38,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742037_1213 (size=7415) 2024-11-10T04:37:39,376 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/d9f0b36ca44f467e8b99c3de7534c603 2024-11-10T04:37:39,381 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/A/e298baa8003d4841a3f3f43fa4fac40a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e298baa8003d4841a3f3f43fa4fac40a 2024-11-10T04:37:39,385 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e298baa8003d4841a3f3f43fa4fac40a, entries=50, sequenceid=346, filesize=13.5 K 2024-11-10T04:37:39,386 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/B/cafaa87cf75b4004a2934c8cf26935bb as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cafaa87cf75b4004a2934c8cf26935bb 2024-11-10T04:37:39,389 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cafaa87cf75b4004a2934c8cf26935bb, entries=50, sequenceid=346, filesize=7.2 K 2024-11-10T04:37:39,390 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/.tmp/C/d9f0b36ca44f467e8b99c3de7534c603 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d9f0b36ca44f467e8b99c3de7534c603 2024-11-10T04:37:39,394 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d9f0b36ca44f467e8b99c3de7534c603, entries=50, sequenceid=346, filesize=7.2 K 2024-11-10T04:37:39,395 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for adb760d52bb12bfd79fa68b6162c27ba in 1659ms, sequenceid=346, compaction requested=false 2024-11-10T04:37:39,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/76689d4e4aeb4d7e8440ffa7c61caebd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/92c41da7e1d24a9686e82c96e2aa43da, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/922008d41cc64a169224b76e2f39e776, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/aeae44ed05454996b9e836d70cf1e2e7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7329709d3d8a41c5b345494f01aa2edc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/62cb3f394e1a4b4ab794aa984e5b2ade, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9624207fa92345088913d71c41584b5d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/6814e12ba0a4485e8de4811d027e77dc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9945c113b79a41e69a898e1841a09e3a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/2c9365fcb62f4a6c9342c3be299dfe46, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5d94ac9598f446f0849af7a60fc1fdd3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5b90b25ee38e4ff592361e0f94fe9713, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9c4e913b524740aea7fa988cd93bbb6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/12d0b60cd83d4ba7bf9099940d8621d2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/91ae7b9919644d0b8eab9eab7031c675, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/a18aa2f17c1f4cecbb6b18210ff2682e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/f990f984d93a4ebdba311e42b36f344a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e2908e4ec95c4d4d87511f0d02082da8, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7245b491668545949ff6db8fa3edd7e5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/cd88871f23c6461ebe5aea233b7c20e7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/b6ade4985f8d458caf5421bef82752ae, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/ad70979812584c8589e6ff736097b549] to archive 2024-11-10T04:37:39,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:37:39,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/76689d4e4aeb4d7e8440ffa7c61caebd to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/76689d4e4aeb4d7e8440ffa7c61caebd 2024-11-10T04:37:39,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/92c41da7e1d24a9686e82c96e2aa43da to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/92c41da7e1d24a9686e82c96e2aa43da 2024-11-10T04:37:39,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/922008d41cc64a169224b76e2f39e776 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/922008d41cc64a169224b76e2f39e776 2024-11-10T04:37:39,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/aeae44ed05454996b9e836d70cf1e2e7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/aeae44ed05454996b9e836d70cf1e2e7 2024-11-10T04:37:39,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7329709d3d8a41c5b345494f01aa2edc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7329709d3d8a41c5b345494f01aa2edc 2024-11-10T04:37:39,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/62cb3f394e1a4b4ab794aa984e5b2ade to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/62cb3f394e1a4b4ab794aa984e5b2ade 2024-11-10T04:37:39,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9624207fa92345088913d71c41584b5d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9624207fa92345088913d71c41584b5d 2024-11-10T04:37:39,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/6814e12ba0a4485e8de4811d027e77dc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/6814e12ba0a4485e8de4811d027e77dc 2024-11-10T04:37:39,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9945c113b79a41e69a898e1841a09e3a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9945c113b79a41e69a898e1841a09e3a 2024-11-10T04:37:39,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/2c9365fcb62f4a6c9342c3be299dfe46 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/2c9365fcb62f4a6c9342c3be299dfe46 2024-11-10T04:37:39,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5d94ac9598f446f0849af7a60fc1fdd3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5d94ac9598f446f0849af7a60fc1fdd3 2024-11-10T04:37:39,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5b90b25ee38e4ff592361e0f94fe9713 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/5b90b25ee38e4ff592361e0f94fe9713 2024-11-10T04:37:39,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9c4e913b524740aea7fa988cd93bbb6f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/9c4e913b524740aea7fa988cd93bbb6f 2024-11-10T04:37:39,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/12d0b60cd83d4ba7bf9099940d8621d2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/12d0b60cd83d4ba7bf9099940d8621d2 2024-11-10T04:37:39,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/91ae7b9919644d0b8eab9eab7031c675 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/91ae7b9919644d0b8eab9eab7031c675 2024-11-10T04:37:39,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/a18aa2f17c1f4cecbb6b18210ff2682e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/a18aa2f17c1f4cecbb6b18210ff2682e 2024-11-10T04:37:39,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/f990f984d93a4ebdba311e42b36f344a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/f990f984d93a4ebdba311e42b36f344a 2024-11-10T04:37:39,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e2908e4ec95c4d4d87511f0d02082da8 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e2908e4ec95c4d4d87511f0d02082da8 2024-11-10T04:37:39,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7245b491668545949ff6db8fa3edd7e5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/7245b491668545949ff6db8fa3edd7e5 2024-11-10T04:37:39,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/cd88871f23c6461ebe5aea233b7c20e7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/cd88871f23c6461ebe5aea233b7c20e7 2024-11-10T04:37:39,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/b6ade4985f8d458caf5421bef82752ae to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/b6ade4985f8d458caf5421bef82752ae 2024-11-10T04:37:39,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/ad70979812584c8589e6ff736097b549 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/ad70979812584c8589e6ff736097b549 2024-11-10T04:37:39,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/c23739ae3cd344ebb8d732c027968373, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/29e4e48246634d7c80ffa7b85f5b7020, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/6f2198e0e29a481f8c128d3f8ae88c23, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/28fd76aca35c47c59bddfeed2b6dbd76, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4f07209a3c3747b1a64535490ab0984c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/13def43cee114737bd7ec9dc4467c12d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7966afbd0bf4f3ea3489e91d1ac4e47, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/ec020211764448939ed2e5546aef2919, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/b19625148d1c40788e11c700756b16a6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e4b17652be5e43349e30770ed0e3489d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4136451a59f745398e0f11369e27453f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2fd65f7c88f14d0baebdf33bbd724035, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2f9b4eda059049719c112605c7dd1ea6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/903e31ba32344b53b443525d955e7078, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/fdc4a97887ac4a23866fa2cb461cca65, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/93b54f88e6c14f4b9d04442df1997464, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e99f80e7fcf4446c9cbafef926ee0804, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/9e8b1061a84a472d997f9d8327b5a35c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/00dceeb7afcd4962b760fe600424301b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cc72dbf99a0047f4a8d18ad5594e8c05, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7af763106964d428e64e2dbcf9f67f1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/5f81934a112e4d16bf1289a00ff52a93] to archive 2024-11-10T04:37:39,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:37:39,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/c23739ae3cd344ebb8d732c027968373 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/c23739ae3cd344ebb8d732c027968373 2024-11-10T04:37:39,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/29e4e48246634d7c80ffa7b85f5b7020 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/29e4e48246634d7c80ffa7b85f5b7020 2024-11-10T04:37:39,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/6f2198e0e29a481f8c128d3f8ae88c23 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/6f2198e0e29a481f8c128d3f8ae88c23 2024-11-10T04:37:39,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/28fd76aca35c47c59bddfeed2b6dbd76 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/28fd76aca35c47c59bddfeed2b6dbd76 2024-11-10T04:37:39,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4f07209a3c3747b1a64535490ab0984c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4f07209a3c3747b1a64535490ab0984c 2024-11-10T04:37:39,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/13def43cee114737bd7ec9dc4467c12d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/13def43cee114737bd7ec9dc4467c12d 2024-11-10T04:37:39,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7966afbd0bf4f3ea3489e91d1ac4e47 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7966afbd0bf4f3ea3489e91d1ac4e47 2024-11-10T04:37:39,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/ec020211764448939ed2e5546aef2919 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/ec020211764448939ed2e5546aef2919 2024-11-10T04:37:39,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/b19625148d1c40788e11c700756b16a6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/b19625148d1c40788e11c700756b16a6 2024-11-10T04:37:39,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e4b17652be5e43349e30770ed0e3489d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e4b17652be5e43349e30770ed0e3489d 2024-11-10T04:37:39,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4136451a59f745398e0f11369e27453f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/4136451a59f745398e0f11369e27453f 2024-11-10T04:37:39,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2fd65f7c88f14d0baebdf33bbd724035 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2fd65f7c88f14d0baebdf33bbd724035 2024-11-10T04:37:39,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2f9b4eda059049719c112605c7dd1ea6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/2f9b4eda059049719c112605c7dd1ea6 2024-11-10T04:37:39,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/903e31ba32344b53b443525d955e7078 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/903e31ba32344b53b443525d955e7078 2024-11-10T04:37:39,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/fdc4a97887ac4a23866fa2cb461cca65 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/fdc4a97887ac4a23866fa2cb461cca65 2024-11-10T04:37:39,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/93b54f88e6c14f4b9d04442df1997464 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/93b54f88e6c14f4b9d04442df1997464 2024-11-10T04:37:39,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e99f80e7fcf4446c9cbafef926ee0804 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/e99f80e7fcf4446c9cbafef926ee0804 2024-11-10T04:37:39,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/9e8b1061a84a472d997f9d8327b5a35c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/9e8b1061a84a472d997f9d8327b5a35c 2024-11-10T04:37:39,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/00dceeb7afcd4962b760fe600424301b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/00dceeb7afcd4962b760fe600424301b 2024-11-10T04:37:39,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cc72dbf99a0047f4a8d18ad5594e8c05 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cc72dbf99a0047f4a8d18ad5594e8c05 2024-11-10T04:37:39,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7af763106964d428e64e2dbcf9f67f1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/d7af763106964d428e64e2dbcf9f67f1 2024-11-10T04:37:39,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/5f81934a112e4d16bf1289a00ff52a93 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/5f81934a112e4d16bf1289a00ff52a93 2024-11-10T04:37:39,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eae0b7f721be41ca8e067bc385ac39e3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/2cd5c46a76cb45bfbefd646107564f24, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/25fe1f979b1b4ae9a987df12aabf5e03, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6afdfed40c844f9faf3edd7e78771cfe, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8bde4762e8c847419f8f8d18e1175fd6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/ca4955d362c74c9e863140855bbb914b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d912af78e4cc4915800b90d3ac2028dc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f1f75e1ed4db4a68b9443ccd66a79b6b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f7bd24dda3854068b680a09589e9922a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/a4f691b41715444bbcd6ab9d31ff6fc4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c3cc2e4de6b24a76abde6bd2a7230691, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4117577bc84541c880dd62cfdb4b34c6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8244d63ca0474537970b8b98f76882e9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4eab94ee38f141f28cfa694af69054ce, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/3cf4474397f5461bb9fe0307bb635f1e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/fc7ba1e08ba04208a33ad06125757535, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6138507c03674dc5978e420e8d57a8b4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/66eec9894e2c4a16af8873a1562596db, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d04556ee15d0403dbdbe49133954c372, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eaddccdd21c84e51864280d7dad85ad0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/581d6cb4030d420d9161afd5987457ac, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c078ce6c5b824ba8b1d66a76b01ea32f] to archive 2024-11-10T04:37:39,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:37:39,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eae0b7f721be41ca8e067bc385ac39e3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eae0b7f721be41ca8e067bc385ac39e3 2024-11-10T04:37:39,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/2cd5c46a76cb45bfbefd646107564f24 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/2cd5c46a76cb45bfbefd646107564f24 2024-11-10T04:37:39,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/25fe1f979b1b4ae9a987df12aabf5e03 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/25fe1f979b1b4ae9a987df12aabf5e03 2024-11-10T04:37:39,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6afdfed40c844f9faf3edd7e78771cfe to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6afdfed40c844f9faf3edd7e78771cfe 2024-11-10T04:37:39,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8bde4762e8c847419f8f8d18e1175fd6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8bde4762e8c847419f8f8d18e1175fd6 2024-11-10T04:37:39,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/ca4955d362c74c9e863140855bbb914b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/ca4955d362c74c9e863140855bbb914b 2024-11-10T04:37:39,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d912af78e4cc4915800b90d3ac2028dc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d912af78e4cc4915800b90d3ac2028dc 2024-11-10T04:37:39,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f1f75e1ed4db4a68b9443ccd66a79b6b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f1f75e1ed4db4a68b9443ccd66a79b6b 2024-11-10T04:37:39,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f7bd24dda3854068b680a09589e9922a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/f7bd24dda3854068b680a09589e9922a 2024-11-10T04:37:39,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/a4f691b41715444bbcd6ab9d31ff6fc4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/a4f691b41715444bbcd6ab9d31ff6fc4 2024-11-10T04:37:39,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c3cc2e4de6b24a76abde6bd2a7230691 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c3cc2e4de6b24a76abde6bd2a7230691 2024-11-10T04:37:39,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4117577bc84541c880dd62cfdb4b34c6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4117577bc84541c880dd62cfdb4b34c6 2024-11-10T04:37:39,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8244d63ca0474537970b8b98f76882e9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/8244d63ca0474537970b8b98f76882e9 2024-11-10T04:37:39,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4eab94ee38f141f28cfa694af69054ce to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/4eab94ee38f141f28cfa694af69054ce 2024-11-10T04:37:39,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/3cf4474397f5461bb9fe0307bb635f1e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/3cf4474397f5461bb9fe0307bb635f1e 2024-11-10T04:37:39,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/fc7ba1e08ba04208a33ad06125757535 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/fc7ba1e08ba04208a33ad06125757535 2024-11-10T04:37:39,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6138507c03674dc5978e420e8d57a8b4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/6138507c03674dc5978e420e8d57a8b4 2024-11-10T04:37:39,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/66eec9894e2c4a16af8873a1562596db to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/66eec9894e2c4a16af8873a1562596db 2024-11-10T04:37:39,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d04556ee15d0403dbdbe49133954c372 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d04556ee15d0403dbdbe49133954c372 2024-11-10T04:37:39,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eaddccdd21c84e51864280d7dad85ad0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/eaddccdd21c84e51864280d7dad85ad0 2024-11-10T04:37:39,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/581d6cb4030d420d9161afd5987457ac to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/581d6cb4030d420d9161afd5987457ac 2024-11-10T04:37:39,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c078ce6c5b824ba8b1d66a76b01ea32f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/c078ce6c5b824ba8b1d66a76b01ea32f 2024-11-10T04:37:39,472 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/recovered.edits/349.seqid, newMaxSeqId=349, maxSeqId=4 2024-11-10T04:37:39,473 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. 2024-11-10T04:37:39,473 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for adb760d52bb12bfd79fa68b6162c27ba: Waiting for close lock at 1731213457735Running coprocessor pre-close hooks at 1731213457735Disabling compacts and flushes for region at 1731213457735Disabling writes for close at 1731213457735Obtaining lock to block concurrent updates at 1731213457735Preparing flush snapshotting stores in adb760d52bb12bfd79fa68b6162c27ba at 1731213457735Finished memstore snapshotting TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731213457735Flushing stores of TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba. at 1731213457735Flushing adb760d52bb12bfd79fa68b6162c27ba/A: creating writer at 1731213457736 (+1 ms)Flushing mob file adb760d52bb12bfd79fa68b6162c27ba/A: appending metadata at 1731213457740 (+4 ms)Flushing mob file adb760d52bb12bfd79fa68b6162c27ba/A: closing flushed file at 1731213457740Flushing adb760d52bb12bfd79fa68b6162c27ba/A: appending metadata at 1731213458149 (+409 ms)Flushing adb760d52bb12bfd79fa68b6162c27ba/A: closing flushed file at 1731213458149Flushing adb760d52bb12bfd79fa68b6162c27ba/B: creating writer at 1731213458558 (+409 ms)Flushing adb760d52bb12bfd79fa68b6162c27ba/B: appending metadata at 1731213458561 (+3 ms)Flushing adb760d52bb12bfd79fa68b6162c27ba/B: closing flushed file at 1731213458561Flushing adb760d52bb12bfd79fa68b6162c27ba/C: creating writer at 1731213458969 (+408 ms)Flushing adb760d52bb12bfd79fa68b6162c27ba/C: appending metadata at 1731213458972 (+3 ms)Flushing adb760d52bb12bfd79fa68b6162c27ba/C: closing flushed file at 1731213458972Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@504cd6ff: reopening flushed file at 1731213459380 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55213a51: reopening flushed file at 1731213459385 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd1df: reopening flushed file at 1731213459390 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for adb760d52bb12bfd79fa68b6162c27ba in 1659ms, sequenceid=346, compaction requested=false at 1731213459395 (+5 ms)Writing region close event to WAL at 1731213459469 (+74 ms)Running coprocessor post-close hooks at 1731213459472 (+3 ms)Closed at 1731213459473 (+1 ms) 2024-11-10T04:37:39,475 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,475 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=adb760d52bb12bfd79fa68b6162c27ba, regionState=CLOSED 2024-11-10T04:37:39,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:37:39,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-10T04:37:39,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure adb760d52bb12bfd79fa68b6162c27ba, server=d1d9e3766cb1,45693,1731213391842 in 1.8940 sec 2024-11-10T04:37:39,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-10T04:37:39,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=adb760d52bb12bfd79fa68b6162c27ba, UNASSIGN in 1.8980 sec 2024-11-10T04:37:39,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-10T04:37:39,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9010 sec 2024-11-10T04:37:39,483 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213459483"}]},"ts":"1731213459483"} 2024-11-10T04:37:39,484 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T04:37:39,484 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T04:37:39,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9110 sec 2024-11-10T04:37:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T04:37:39,721 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:39,722 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T04:37:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:39,723 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T04:37:39,724 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:39,726 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,728 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/recovered.edits] 2024-11-10T04:37:39,731 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/64ae4c5fb7d144be9c65e5f2da479286 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/64ae4c5fb7d144be9c65e5f2da479286 2024-11-10T04:37:39,732 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e298baa8003d4841a3f3f43fa4fac40a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/A/e298baa8003d4841a3f3f43fa4fac40a 2024-11-10T04:37:39,735 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/b4c4184ca54b403e9b417fbc9ce0e6be to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/b4c4184ca54b403e9b417fbc9ce0e6be 2024-11-10T04:37:39,736 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cafaa87cf75b4004a2934c8cf26935bb to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/B/cafaa87cf75b4004a2934c8cf26935bb 2024-11-10T04:37:39,738 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/1ec6fe7cb5464390a3a4d60011069a4b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/1ec6fe7cb5464390a3a4d60011069a4b 2024-11-10T04:37:39,739 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d9f0b36ca44f467e8b99c3de7534c603 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/C/d9f0b36ca44f467e8b99c3de7534c603 2024-11-10T04:37:39,741 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/recovered.edits/349.seqid to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba/recovered.edits/349.seqid 2024-11-10T04:37:39,742 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,742 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T04:37:39,742 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T04:37:39,743 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T04:37:39,746 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110050879b874b84f0a9fa0ba63ac870892_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110050879b874b84f0a9fa0ba63ac870892_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,747 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110167ae4c85c9b4d61b02348500dc6d5ae_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110167ae4c85c9b4d61b02348500dc6d5ae_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,749 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101f5f9809009e4578a703debbac2664b8_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101f5f9809009e4578a703debbac2664b8_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,750 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103bac530d0a2542d6af1e1540280a53f0_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103bac530d0a2542d6af1e1540280a53f0_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,751 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103db23dc8661344e5afb3a604351212f9_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103db23dc8661344e5afb3a604351212f9_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,752 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e99887c6b144ef1b78e3f6d949a9a02_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104e99887c6b144ef1b78e3f6d949a9a02_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,753 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110527a9d7fb7d142d883c3d5718427fd94_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110527a9d7fb7d142d883c3d5718427fd94_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,754 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a21befeb6e24b338907064f0c0d6fad_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a21befeb6e24b338907064f0c0d6fad_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,755 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107bd0a7e6067440af8b9ff6be4d3c3642_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107bd0a7e6067440af8b9ff6be4d3c3642_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,756 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110810697ec88fa472393b513528ff94a82_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110810697ec88fa472393b513528ff94a82_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,757 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111094ea5c3b20f248bdb54e06927d562b18_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111094ea5c3b20f248bdb54e06927d562b18_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,759 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a566d05e4b42442baf4d02d795edeaf7_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a566d05e4b42442baf4d02d795edeaf7_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,760 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a5cc7485abfd4588830b46d540703f3f_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a5cc7485abfd4588830b46d540703f3f_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,761 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ab069c9eaf82412ea6f460447d45f046_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ab069c9eaf82412ea6f460447d45f046_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,762 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c5254bec3ed8494097837ccae3b7af09_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c5254bec3ed8494097837ccae3b7af09_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,763 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cd9931747ac24fafb7ce9644aa533d00_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cd9931747ac24fafb7ce9644aa533d00_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,765 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d290b655b2074c30b595a130fa980f78_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d290b655b2074c30b595a130fa980f78_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,766 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e8961f6e92c1440dab2a20814c63024e_adb760d52bb12bfd79fa68b6162c27ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e8961f6e92c1440dab2a20814c63024e_adb760d52bb12bfd79fa68b6162c27ba 2024-11-10T04:37:39,766 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T04:37:39,769 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:39,771 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T04:37:39,773 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T04:37:39,774 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:39,774 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T04:37:39,774 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731213459774"}]},"ts":"9223372036854775807"} 2024-11-10T04:37:39,776 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T04:37:39,776 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => adb760d52bb12bfd79fa68b6162c27ba, NAME => 'TestAcidGuarantees,,1731213430417.adb760d52bb12bfd79fa68b6162c27ba.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T04:37:39,776 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T04:37:39,776 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731213459776"}]},"ts":"9223372036854775807"} 2024-11-10T04:37:39,778 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T04:37:39,778 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:39,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-10T04:37:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T04:37:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T04:37:39,831 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:39,841 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1218819243_22 at /127.0.0.1:45084 [Waiting for operation #741] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-554903022_22 at /127.0.0.1:45022 [Waiting for operation #763] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-554903022_22 at /127.0.0.1:45068 [Waiting for operation #745] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1218819243_22 at /127.0.0.1:42810 [Waiting for operation #773] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=214 (was 196) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6468 (was 6603) 2024-11-10T04:37:39,850 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=6467 2024-11-10T04:37:39,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T04:37:39,852 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:37:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T04:37:39,854 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:37:39,854 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:39,854 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-10T04:37:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T04:37:39,855 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:37:39,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742038_1214 (size=959) 2024-11-10T04:37:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T04:37:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T04:37:40,263 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8 2024-11-10T04:37:40,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742039_1215 (size=53) 2024-11-10T04:37:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T04:37:40,669 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:37:40,669 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 305ea5ff5a8e2066f94db3068ceb2842, disabling compactions & flushes 2024-11-10T04:37:40,669 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:40,669 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:40,669 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. after waiting 0 ms 2024-11-10T04:37:40,669 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:40,669 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:40,669 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 305ea5ff5a8e2066f94db3068ceb2842: Waiting for close lock at 1731213460669Disabling compacts and flushes for region at 1731213460669Disabling writes for close at 1731213460669Writing region close event to WAL at 1731213460669Closed at 1731213460669 2024-11-10T04:37:40,670 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:37:40,670 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731213460670"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731213460670"}]},"ts":"1731213460670"} 2024-11-10T04:37:40,672 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:37:40,672 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:37:40,672 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213460672"}]},"ts":"1731213460672"} 2024-11-10T04:37:40,674 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T04:37:40,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ea5ff5a8e2066f94db3068ceb2842, ASSIGN}] 2024-11-10T04:37:40,675 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ea5ff5a8e2066f94db3068ceb2842, ASSIGN 2024-11-10T04:37:40,675 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ea5ff5a8e2066f94db3068ceb2842, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,45693,1731213391842; forceNewPlan=false, retain=false 2024-11-10T04:37:40,826 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=305ea5ff5a8e2066f94db3068ceb2842, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:40,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ea5ff5a8e2066f94db3068ceb2842, ASSIGN because future has completed 2024-11-10T04:37:40,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:37:40,982 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:40,983 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:37:40,983 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,983 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:37:40,983 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,983 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,984 INFO [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,986 INFO [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 305ea5ff5a8e2066f94db3068ceb2842 columnFamilyName A 2024-11-10T04:37:40,986 DEBUG [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:40,986 INFO [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] regionserver.HStore(327): Store=305ea5ff5a8e2066f94db3068ceb2842/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:37:40,986 INFO [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,987 INFO [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 305ea5ff5a8e2066f94db3068ceb2842 columnFamilyName B 2024-11-10T04:37:40,987 DEBUG [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:40,988 INFO [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] regionserver.HStore(327): Store=305ea5ff5a8e2066f94db3068ceb2842/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:37:40,988 INFO [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,989 INFO [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 305ea5ff5a8e2066f94db3068ceb2842 columnFamilyName C 2024-11-10T04:37:40,989 DEBUG [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:37:40,989 INFO [StoreOpener-305ea5ff5a8e2066f94db3068ceb2842-1 {}] regionserver.HStore(327): Store=305ea5ff5a8e2066f94db3068ceb2842/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:37:40,989 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,990 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,990 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T04:37:40,991 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,991 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,991 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:37:40,993 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,994 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:37:40,995 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 305ea5ff5a8e2066f94db3068ceb2842; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72239604, jitterRate=0.07645398378372192}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:37:40,995 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:40,995 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 305ea5ff5a8e2066f94db3068ceb2842: Running coprocessor pre-open hook at 1731213460983Writing region info on filesystem at 1731213460983Initializing all the Stores at 1731213460984 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213460984Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213460984Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213460984Cleaning up temporary data from old regions at 1731213460991 (+7 ms)Running coprocessor post-open hooks at 1731213460995 (+4 ms)Region opened successfully at 1731213460995 2024-11-10T04:37:40,996 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., pid=65, masterSystemTime=1731213460979 2024-11-10T04:37:40,997 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:40,997 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:40,998 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=305ea5ff5a8e2066f94db3068ceb2842, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:40,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:37:41,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-10T04:37:41,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 in 172 msec 2024-11-10T04:37:41,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-10T04:37:41,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ea5ff5a8e2066f94db3068ceb2842, ASSIGN in 327 msec 2024-11-10T04:37:41,003 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:37:41,003 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213461003"}]},"ts":"1731213461003"} 2024-11-10T04:37:41,005 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T04:37:41,005 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:37:41,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-10T04:37:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T04:37:42,001 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:42,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,003 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc4f873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,005 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18884b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,006 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,008 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:37:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,42929,1731213391190 2024-11-10T04:37:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27893236 2024-11-10T04:37:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:37:42,010 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:37:42,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,012 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43aea78f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,013 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@659b74c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,015 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b1199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,016 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b57881c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,017 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac71e04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,018 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d691b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,020 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e43d670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4313f936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,023 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148a03ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,024 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74808c51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,026 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3f2ed7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,027 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e220f52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,028 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6afd2f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a51e87f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,031 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6e9ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,032 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e32581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,034 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13835e08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,035 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ec024e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:37:42,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:37:42,036 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:37:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:37:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:37:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58eb6593, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:37:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:37:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:37:42,037 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:37:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a50dc94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:37:42,038 DEBUG [Thread-1034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,038 DEBUG [Thread-1025 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,038 DEBUG [Thread-1032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,039 DEBUG [Thread-1038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,039 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:42,039 DEBUG [Thread-1021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,039 DEBUG [Thread-1040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,040 DEBUG [Thread-1023 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-10T04:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,041 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T04:37:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,043 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:42,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,043 DEBUG [Thread-1027 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,043 DEBUG [Thread-1036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:42,043 DEBUG [Thread-1029 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:37:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:37:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:37:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:37:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:37:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:42,081 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:42,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:37:42,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/8bb1c97e732d46d38cdfddf07414ae63 is 50, key is test_row_0/A:col10/1731213462049/Put/seqid=0 2024-11-10T04:37:42,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213522115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213522116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213522117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213522117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213522119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742040_1216 (size=12001) 2024-11-10T04:37:42,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/8bb1c97e732d46d38cdfddf07414ae63 2024-11-10T04:37:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T04:37:42,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/d7908dc2e7d44544bc7ab10ea4cba02b is 50, key is test_row_0/B:col10/1731213462049/Put/seqid=0 2024-11-10T04:37:42,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742041_1217 (size=12001) 2024-11-10T04:37:42,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/d7908dc2e7d44544bc7ab10ea4cba02b 2024-11-10T04:37:42,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T04:37:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:42,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:42,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/a98f040ba7b442fa96642c0d312e8e77 is 50, key is test_row_0/C:col10/1731213462049/Put/seqid=0 2024-11-10T04:37:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213522231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213522231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213522231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213522232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213522232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742042_1218 (size=12001) 2024-11-10T04:37:42,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/a98f040ba7b442fa96642c0d312e8e77 2024-11-10T04:37:42,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/8bb1c97e732d46d38cdfddf07414ae63 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/8bb1c97e732d46d38cdfddf07414ae63 2024-11-10T04:37:42,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/8bb1c97e732d46d38cdfddf07414ae63, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T04:37:42,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/d7908dc2e7d44544bc7ab10ea4cba02b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d7908dc2e7d44544bc7ab10ea4cba02b 2024-11-10T04:37:42,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d7908dc2e7d44544bc7ab10ea4cba02b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T04:37:42,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/a98f040ba7b442fa96642c0d312e8e77 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/a98f040ba7b442fa96642c0d312e8e77 2024-11-10T04:37:42,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/a98f040ba7b442fa96642c0d312e8e77, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T04:37:42,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 305ea5ff5a8e2066f94db3068ceb2842 in 200ms, sequenceid=13, compaction requested=false 2024-11-10T04:37:42,281 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T04:37:42,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:42,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T04:37:42,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:42,350 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T04:37:42,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/f91922f4fe4f448ebfec740e0cd4895a is 50, key is test_row_0/A:col10/1731213462117/Put/seqid=0 2024-11-10T04:37:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T04:37:42,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742043_1219 (size=12001) 2024-11-10T04:37:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:42,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:42,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213522446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213522450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213522451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213522453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213522453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213522561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213522561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213522561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213522561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213522561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T04:37:42,771 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/f91922f4fe4f448ebfec740e0cd4895a 2024-11-10T04:37:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213522771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213522771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213522771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213522771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213522771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/5b0cad92709845a4b1fe1a852087f69f is 50, key is test_row_0/B:col10/1731213462117/Put/seqid=0 2024-11-10T04:37:42,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742044_1220 (size=12001) 2024-11-10T04:37:43,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213523081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213523081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213523081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213523081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213523081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T04:37:43,187 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/5b0cad92709845a4b1fe1a852087f69f 2024-11-10T04:37:43,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4 is 50, key is test_row_0/C:col10/1731213462117/Put/seqid=0 2024-11-10T04:37:43,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742045_1221 (size=12001) 2024-11-10T04:37:43,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213523591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213523591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213523591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213523591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213523591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:43,604 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4 2024-11-10T04:37:43,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/f91922f4fe4f448ebfec740e0cd4895a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f91922f4fe4f448ebfec740e0cd4895a 2024-11-10T04:37:43,613 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f91922f4fe4f448ebfec740e0cd4895a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T04:37:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/5b0cad92709845a4b1fe1a852087f69f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/5b0cad92709845a4b1fe1a852087f69f 2024-11-10T04:37:43,618 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/5b0cad92709845a4b1fe1a852087f69f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T04:37:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4 2024-11-10T04:37:43,623 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T04:37:43,624 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 305ea5ff5a8e2066f94db3068ceb2842 in 1274ms, sequenceid=37, compaction requested=false 2024-11-10T04:37:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-10T04:37:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-10T04:37:43,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T04:37:43,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5820 sec 2024-11-10T04:37:43,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5890 sec 2024-11-10T04:37:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T04:37:44,191 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:44,192 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T04:37:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T04:37:44,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:44,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:44,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T04:37:44,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T04:37:44,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:44,347 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:37:44,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/a03d0b38999548c8b88f05929822ec76 is 50, key is test_row_0/A:col10/1731213462449/Put/seqid=0 2024-11-10T04:37:44,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742046_1222 (size=12001) 2024-11-10T04:37:44,357 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/a03d0b38999548c8b88f05929822ec76 2024-11-10T04:37:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/31c135312d8d461f9a85080c497b501e is 50, key is test_row_0/B:col10/1731213462449/Put/seqid=0 2024-11-10T04:37:44,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742047_1223 (size=12001) 2024-11-10T04:37:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T04:37:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:44,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213524629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213524631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213524630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213524632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213524632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213524741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213524741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213524741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213524741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213524741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,776 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/31c135312d8d461f9a85080c497b501e 2024-11-10T04:37:44,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/0774fecf2dc8437d9c522c08169de634 is 50, key is test_row_0/C:col10/1731213462449/Put/seqid=0 2024-11-10T04:37:44,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742048_1224 (size=12001) 2024-11-10T04:37:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T04:37:44,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213524951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213524951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213524951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213524951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213524951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:44,998 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T04:37:45,190 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/0774fecf2dc8437d9c522c08169de634 2024-11-10T04:37:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/a03d0b38999548c8b88f05929822ec76 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a03d0b38999548c8b88f05929822ec76 2024-11-10T04:37:45,200 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a03d0b38999548c8b88f05929822ec76, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T04:37:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/31c135312d8d461f9a85080c497b501e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/31c135312d8d461f9a85080c497b501e 2024-11-10T04:37:45,205 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/31c135312d8d461f9a85080c497b501e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T04:37:45,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/0774fecf2dc8437d9c522c08169de634 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0774fecf2dc8437d9c522c08169de634 2024-11-10T04:37:45,210 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0774fecf2dc8437d9c522c08169de634, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T04:37:45,211 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 305ea5ff5a8e2066f94db3068ceb2842 in 864ms, sequenceid=49, compaction requested=true 2024-11-10T04:37:45,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:45,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:45,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T04:37:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T04:37:45,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T04:37:45,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-10T04:37:45,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0230 sec 2024-11-10T04:37:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:45,262 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:45,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:37:45,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/19bdedf0e291431a82bf00fa35f71db0 is 50, key is test_row_0/A:col10/1731213465261/Put/seqid=0 2024-11-10T04:37:45,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213525265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742049_1225 (size=12001) 2024-11-10T04:37:45,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/19bdedf0e291431a82bf00fa35f71db0 2024-11-10T04:37:45,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213525270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213525270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213525272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213525273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/11d1d0d24cba4befbf77bf9e3741df93 is 50, key is test_row_0/B:col10/1731213465261/Put/seqid=0 2024-11-10T04:37:45,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742050_1226 (size=12001) 2024-11-10T04:37:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T04:37:45,331 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:45,331 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T04:37:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T04:37:45,333 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:45,334 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:45,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213525381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213525381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213525381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213525381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213525382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T04:37:45,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T04:37:45,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:45,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:45,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:45,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:45,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213525591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213525592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213525592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213525592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213525592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T04:37:45,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:45,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:45,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:45,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:45,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T04:37:45,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/11d1d0d24cba4befbf77bf9e3741df93 2024-11-10T04:37:45,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/8cde5156cff749a6b9819d240683f236 is 50, key is test_row_0/C:col10/1731213465261/Put/seqid=0 2024-11-10T04:37:45,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742051_1227 (size=12001) 2024-11-10T04:37:45,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/8cde5156cff749a6b9819d240683f236 2024-11-10T04:37:45,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/19bdedf0e291431a82bf00fa35f71db0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/19bdedf0e291431a82bf00fa35f71db0 2024-11-10T04:37:45,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/19bdedf0e291431a82bf00fa35f71db0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T04:37:45,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/11d1d0d24cba4befbf77bf9e3741df93 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/11d1d0d24cba4befbf77bf9e3741df93 2024-11-10T04:37:45,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/11d1d0d24cba4befbf77bf9e3741df93, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T04:37:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/8cde5156cff749a6b9819d240683f236 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8cde5156cff749a6b9819d240683f236 2024-11-10T04:37:45,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8cde5156cff749a6b9819d240683f236, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T04:37:45,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 305ea5ff5a8e2066f94db3068ceb2842 in 464ms, sequenceid=76, compaction requested=true 2024-11-10T04:37:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:45,726 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:45,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:45,726 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:45,727 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:45,727 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:45,728 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/A is initiating minor compaction (all files) 2024-11-10T04:37:45,728 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/B is initiating minor compaction (all files) 2024-11-10T04:37:45,728 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/A in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:45,728 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/B in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:45,728 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/8bb1c97e732d46d38cdfddf07414ae63, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f91922f4fe4f448ebfec740e0cd4895a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a03d0b38999548c8b88f05929822ec76, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/19bdedf0e291431a82bf00fa35f71db0] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=46.9 K 2024-11-10T04:37:45,728 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d7908dc2e7d44544bc7ab10ea4cba02b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/5b0cad92709845a4b1fe1a852087f69f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/31c135312d8d461f9a85080c497b501e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/11d1d0d24cba4befbf77bf9e3741df93] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=46.9 K 2024-11-10T04:37:45,729 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d7908dc2e7d44544bc7ab10ea4cba02b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731213462049 2024-11-10T04:37:45,729 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb1c97e732d46d38cdfddf07414ae63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731213462049 2024-11-10T04:37:45,729 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b0cad92709845a4b1fe1a852087f69f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731213462113 2024-11-10T04:37:45,729 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f91922f4fe4f448ebfec740e0cd4895a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731213462113 2024-11-10T04:37:45,730 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 31c135312d8d461f9a85080c497b501e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731213462443 2024-11-10T04:37:45,730 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a03d0b38999548c8b88f05929822ec76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731213462443 2024-11-10T04:37:45,730 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 11d1d0d24cba4befbf77bf9e3741df93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731213464624 2024-11-10T04:37:45,730 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19bdedf0e291431a82bf00fa35f71db0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731213464624 2024-11-10T04:37:45,744 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:45,744 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/4f3026b4dc574586b0e87aed4bc86070 is 50, key is test_row_0/B:col10/1731213465261/Put/seqid=0 2024-11-10T04:37:45,747 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#A#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:45,747 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/489a278991f246ddba3865bab93d1d74 is 50, key is test_row_0/A:col10/1731213465261/Put/seqid=0 2024-11-10T04:37:45,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742052_1228 (size=12139) 2024-11-10T04:37:45,763 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/4f3026b4dc574586b0e87aed4bc86070 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4f3026b4dc574586b0e87aed4bc86070 2024-11-10T04:37:45,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742053_1229 (size=12139) 2024-11-10T04:37:45,776 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/B of 305ea5ff5a8e2066f94db3068ceb2842 into 4f3026b4dc574586b0e87aed4bc86070(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:45,776 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:45,776 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/B, priority=12, startTime=1731213465726; duration=0sec 2024-11-10T04:37:45,776 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:45,776 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:B 2024-11-10T04:37:45,776 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:45,778 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:45,778 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/C is initiating minor compaction (all files) 2024-11-10T04:37:45,778 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/C in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:45,778 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/a98f040ba7b442fa96642c0d312e8e77, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0774fecf2dc8437d9c522c08169de634, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8cde5156cff749a6b9819d240683f236] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=46.9 K 2024-11-10T04:37:45,779 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a98f040ba7b442fa96642c0d312e8e77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731213462049 2024-11-10T04:37:45,779 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731213462113 2024-11-10T04:37:45,780 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0774fecf2dc8437d9c522c08169de634, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731213462443 2024-11-10T04:37:45,780 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cde5156cff749a6b9819d240683f236, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731213464624 2024-11-10T04:37:45,789 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#C#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:45,789 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/b14e83c03bd845f0b66447258c73877b is 50, key is test_row_0/C:col10/1731213465261/Put/seqid=0 2024-11-10T04:37:45,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T04:37:45,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:45,793 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T04:37:45,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742054_1230 (size=12139) 2024-11-10T04:37:45,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/6ed60759f7de45e6afc4feb34218ee01 is 50, key is test_row_0/A:col10/1731213465268/Put/seqid=0 2024-11-10T04:37:45,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742055_1231 (size=12001) 2024-11-10T04:37:45,802 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/6ed60759f7de45e6afc4feb34218ee01 2024-11-10T04:37:45,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/8088d2e1800d4a21bb06c92fcb0d9dba is 50, key is test_row_0/B:col10/1731213465268/Put/seqid=0 2024-11-10T04:37:45,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742056_1232 (size=12001) 2024-11-10T04:37:45,824 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/8088d2e1800d4a21bb06c92fcb0d9dba 2024-11-10T04:37:45,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/1f175955c0084987b730522ec1bf5763 is 50, key is test_row_0/C:col10/1731213465268/Put/seqid=0 2024-11-10T04:37:45,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742057_1233 (size=12001) 2024-11-10T04:37:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:45,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:45,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213525925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213525926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213525927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213525929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213525932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T04:37:46,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213526041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213526041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213526041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213526042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213526042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,178 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/489a278991f246ddba3865bab93d1d74 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/489a278991f246ddba3865bab93d1d74 2024-11-10T04:37:46,184 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/A of 305ea5ff5a8e2066f94db3068ceb2842 into 489a278991f246ddba3865bab93d1d74(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:46,184 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:46,184 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/A, priority=12, startTime=1731213465726; duration=0sec 2024-11-10T04:37:46,184 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:46,184 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:A 2024-11-10T04:37:46,198 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/b14e83c03bd845f0b66447258c73877b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b14e83c03bd845f0b66447258c73877b 2024-11-10T04:37:46,204 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/C of 305ea5ff5a8e2066f94db3068ceb2842 into b14e83c03bd845f0b66447258c73877b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:46,204 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:46,204 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/C, priority=12, startTime=1731213465726; duration=0sec 2024-11-10T04:37:46,204 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:46,204 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:C 2024-11-10T04:37:46,237 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/1f175955c0084987b730522ec1bf5763 2024-11-10T04:37:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/6ed60759f7de45e6afc4feb34218ee01 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6ed60759f7de45e6afc4feb34218ee01 2024-11-10T04:37:46,245 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6ed60759f7de45e6afc4feb34218ee01, entries=150, sequenceid=87, filesize=11.7 K 2024-11-10T04:37:46,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/8088d2e1800d4a21bb06c92fcb0d9dba as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8088d2e1800d4a21bb06c92fcb0d9dba 2024-11-10T04:37:46,250 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8088d2e1800d4a21bb06c92fcb0d9dba, entries=150, sequenceid=87, filesize=11.7 K 2024-11-10T04:37:46,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/1f175955c0084987b730522ec1bf5763 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/1f175955c0084987b730522ec1bf5763 2024-11-10T04:37:46,255 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/1f175955c0084987b730522ec1bf5763, entries=150, sequenceid=87, filesize=11.7 K 2024-11-10T04:37:46,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213526251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,256 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 305ea5ff5a8e2066f94db3068ceb2842 in 463ms, sequenceid=87, compaction requested=false 2024-11-10T04:37:46,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:46,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:46,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T04:37:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T04:37:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:46,257 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:46,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T04:37:46,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T04:37:46,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 924 msec 2024-11-10T04:37:46,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/a79c8a40cd7c49a0975f0a005c923611 is 50, key is test_row_0/A:col10/1731213465923/Put/seqid=0 2024-11-10T04:37:46,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 930 msec 2024-11-10T04:37:46,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742058_1234 (size=12001) 2024-11-10T04:37:46,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/a79c8a40cd7c49a0975f0a005c923611 2024-11-10T04:37:46,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213526263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213526265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213526266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213526267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/57fbee14ea9f49c59674d281fcff1b09 is 50, key is test_row_0/B:col10/1731213465923/Put/seqid=0 2024-11-10T04:37:46,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742059_1235 (size=12001) 2024-11-10T04:37:46,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/57fbee14ea9f49c59674d281fcff1b09 2024-11-10T04:37:46,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2d1250834dcc439aba5424ca4085ee46 is 50, key is test_row_0/C:col10/1731213465923/Put/seqid=0 2024-11-10T04:37:46,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742060_1236 (size=12001) 2024-11-10T04:37:46,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213526371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213526381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213526381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213526381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T04:37:46,471 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:46,472 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-10T04:37:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T04:37:46,474 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:46,474 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:46,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:46,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213526561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T04:37:46,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213526581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213526591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213526592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213526592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T04:37:46,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:46,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:46,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:46,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:46,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:46,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2d1250834dcc439aba5424ca4085ee46 2024-11-10T04:37:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/a79c8a40cd7c49a0975f0a005c923611 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a79c8a40cd7c49a0975f0a005c923611 2024-11-10T04:37:46,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a79c8a40cd7c49a0975f0a005c923611, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T04:37:46,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/57fbee14ea9f49c59674d281fcff1b09 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/57fbee14ea9f49c59674d281fcff1b09 2024-11-10T04:37:46,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/57fbee14ea9f49c59674d281fcff1b09, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T04:37:46,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2d1250834dcc439aba5424ca4085ee46 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2d1250834dcc439aba5424ca4085ee46 2024-11-10T04:37:46,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2d1250834dcc439aba5424ca4085ee46, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T04:37:46,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 305ea5ff5a8e2066f94db3068ceb2842 in 454ms, sequenceid=116, compaction requested=true 2024-11-10T04:37:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:46,711 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:46,712 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:46,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:46,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:46,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:46,713 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:46,713 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:46,713 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/B is initiating minor compaction (all files) 2024-11-10T04:37:46,713 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/A is initiating minor compaction (all files) 2024-11-10T04:37:46,713 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/B in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:46,713 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/A in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:46,713 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4f3026b4dc574586b0e87aed4bc86070, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8088d2e1800d4a21bb06c92fcb0d9dba, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/57fbee14ea9f49c59674d281fcff1b09] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=35.3 K 2024-11-10T04:37:46,713 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/489a278991f246ddba3865bab93d1d74, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6ed60759f7de45e6afc4feb34218ee01, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a79c8a40cd7c49a0975f0a005c923611] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=35.3 K 2024-11-10T04:37:46,714 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3026b4dc574586b0e87aed4bc86070, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731213464624 2024-11-10T04:37:46,714 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 489a278991f246ddba3865bab93d1d74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731213464624 2024-11-10T04:37:46,714 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8088d2e1800d4a21bb06c92fcb0d9dba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731213465263 2024-11-10T04:37:46,714 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ed60759f7de45e6afc4feb34218ee01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731213465263 2024-11-10T04:37:46,714 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 57fbee14ea9f49c59674d281fcff1b09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213465923 2024-11-10T04:37:46,715 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a79c8a40cd7c49a0975f0a005c923611, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213465923 2024-11-10T04:37:46,724 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:46,724 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:46,724 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/a86f16f465674f6eadb5374a26b9fd8a is 50, key is test_row_0/B:col10/1731213465923/Put/seqid=0 2024-11-10T04:37:46,725 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/87c09918febe428cbe01e1e5b1aed8b5 is 50, key is test_row_0/A:col10/1731213465923/Put/seqid=0 2024-11-10T04:37:46,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742061_1237 (size=12241) 2024-11-10T04:37:46,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742062_1238 (size=12241) 2024-11-10T04:37:46,745 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/87c09918febe428cbe01e1e5b1aed8b5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/87c09918febe428cbe01e1e5b1aed8b5 2024-11-10T04:37:46,746 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/a86f16f465674f6eadb5374a26b9fd8a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a86f16f465674f6eadb5374a26b9fd8a 2024-11-10T04:37:46,752 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/B of 305ea5ff5a8e2066f94db3068ceb2842 into a86f16f465674f6eadb5374a26b9fd8a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:46,752 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/A of 305ea5ff5a8e2066f94db3068ceb2842 into 87c09918febe428cbe01e1e5b1aed8b5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:46,752 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:46,752 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:46,752 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/B, priority=13, startTime=1731213466711; duration=0sec 2024-11-10T04:37:46,752 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/A, priority=13, startTime=1731213466711; duration=0sec 2024-11-10T04:37:46,752 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:46,752 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:B 2024-11-10T04:37:46,752 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:46,752 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:A 2024-11-10T04:37:46,752 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:46,753 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:46,753 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/C is initiating minor compaction (all files) 2024-11-10T04:37:46,753 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/C in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:46,753 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b14e83c03bd845f0b66447258c73877b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/1f175955c0084987b730522ec1bf5763, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2d1250834dcc439aba5424ca4085ee46] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=35.3 K 2024-11-10T04:37:46,754 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b14e83c03bd845f0b66447258c73877b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731213464624 2024-11-10T04:37:46,754 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f175955c0084987b730522ec1bf5763, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731213465263 2024-11-10T04:37:46,755 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d1250834dcc439aba5424ca4085ee46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213465923 2024-11-10T04:37:46,764 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:46,765 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/5375759b318748eaaff9975d0d929b2a is 50, key is test_row_0/C:col10/1731213465923/Put/seqid=0 2024-11-10T04:37:46,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742063_1239 (size=12241) 2024-11-10T04:37:46,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T04:37:46,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:46,780 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T04:37:46,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/7f26a420f1ec4bc987617032e1152fc9 is 50, key is test_row_0/A:col10/1731213466258/Put/seqid=0 2024-11-10T04:37:46,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742064_1240 (size=12001) 2024-11-10T04:37:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T04:37:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:46,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:46,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213526927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213526930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213526931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213526941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213527041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213527041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213527041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213527051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213527071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T04:37:47,176 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/5375759b318748eaaff9975d0d929b2a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5375759b318748eaaff9975d0d929b2a 2024-11-10T04:37:47,181 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/C of 305ea5ff5a8e2066f94db3068ceb2842 into 5375759b318748eaaff9975d0d929b2a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:47,181 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:47,181 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/C, priority=13, startTime=1731213466712; duration=0sec 2024-11-10T04:37:47,181 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:47,181 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:C 2024-11-10T04:37:47,190 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/7f26a420f1ec4bc987617032e1152fc9 2024-11-10T04:37:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/d83ca5fc630b4fdc9044f9e36fe8f768 is 50, key is test_row_0/B:col10/1731213466258/Put/seqid=0 2024-11-10T04:37:47,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742065_1241 (size=12001) 2024-11-10T04:37:47,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213527251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213527251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213527251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213527261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213527561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213527561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213527561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213527571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:47,602 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/d83ca5fc630b4fdc9044f9e36fe8f768 2024-11-10T04:37:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T04:37:47,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/d3b335108ea34081883e381719b6b141 is 50, key is test_row_0/C:col10/1731213466258/Put/seqid=0 2024-11-10T04:37:47,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742066_1242 (size=12001) 2024-11-10T04:37:47,616 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/d3b335108ea34081883e381719b6b141 2024-11-10T04:37:47,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/7f26a420f1ec4bc987617032e1152fc9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7f26a420f1ec4bc987617032e1152fc9 2024-11-10T04:37:47,626 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7f26a420f1ec4bc987617032e1152fc9, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T04:37:47,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/d83ca5fc630b4fdc9044f9e36fe8f768 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d83ca5fc630b4fdc9044f9e36fe8f768 2024-11-10T04:37:47,632 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d83ca5fc630b4fdc9044f9e36fe8f768, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T04:37:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/d3b335108ea34081883e381719b6b141 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/d3b335108ea34081883e381719b6b141 2024-11-10T04:37:47,637 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/d3b335108ea34081883e381719b6b141, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T04:37:47,638 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 305ea5ff5a8e2066f94db3068ceb2842 in 857ms, sequenceid=128, compaction requested=false 2024-11-10T04:37:47,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:47,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:47,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-10T04:37:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-10T04:37:47,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T04:37:47,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1650 sec 2024-11-10T04:37:47,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1700 sec 2024-11-10T04:37:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:48,074 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:48,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-10T04:37:48,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/86a9f758380a437680f518388c8fa26e is 50, key is test_row_0/A:col10/1731213468073/Put/seqid=0 2024-11-10T04:37:48,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213528079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213528081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213528081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213528081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742067_1243 (size=14541) 2024-11-10T04:37:48,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213528091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,094 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213528191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213528192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213528192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213528402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213528402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213528402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/86a9f758380a437680f518388c8fa26e 2024-11-10T04:37:48,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/e9780db6b48d42548d5f4c9975018c36 is 50, key is test_row_0/B:col10/1731213468073/Put/seqid=0 2024-11-10T04:37:48,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742068_1244 (size=12151) 2024-11-10T04:37:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T04:37:48,622 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:48,622 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-10T04:37:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T04:37:48,624 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:48,624 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:48,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:48,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213528711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213528711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213528711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:48,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T04:37:48,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T04:37:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:48,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:48,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:48,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/e9780db6b48d42548d5f4c9975018c36 2024-11-10T04:37:48,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/302c3bc1307847afb2c687f62560f299 is 50, key is test_row_0/C:col10/1731213468073/Put/seqid=0 2024-11-10T04:37:48,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742069_1245 (size=12151) 2024-11-10T04:37:48,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T04:37:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:48,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:48,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T04:37:49,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T04:37:49,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:49,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:49,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:49,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:49,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:49,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213529091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:49,095 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213529221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:49,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213529222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:49,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213529222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:49,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T04:37:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:49,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:49,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T04:37:49,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/302c3bc1307847afb2c687f62560f299 2024-11-10T04:37:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/86a9f758380a437680f518388c8fa26e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/86a9f758380a437680f518388c8fa26e 2024-11-10T04:37:49,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/86a9f758380a437680f518388c8fa26e, entries=200, sequenceid=159, filesize=14.2 K 2024-11-10T04:37:49,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/e9780db6b48d42548d5f4c9975018c36 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e9780db6b48d42548d5f4c9975018c36 2024-11-10T04:37:49,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e9780db6b48d42548d5f4c9975018c36, entries=150, sequenceid=159, filesize=11.9 K 2024-11-10T04:37:49,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/302c3bc1307847afb2c687f62560f299 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/302c3bc1307847afb2c687f62560f299 2024-11-10T04:37:49,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/302c3bc1307847afb2c687f62560f299, entries=150, sequenceid=159, filesize=11.9 K 2024-11-10T04:37:49,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 305ea5ff5a8e2066f94db3068ceb2842 in 1262ms, sequenceid=159, compaction requested=true 2024-11-10T04:37:49,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:49,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:49,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:49,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:49,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:49,336 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:49,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:49,336 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:49,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:49,337 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:49,337 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:49,337 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/A is initiating minor compaction (all files) 2024-11-10T04:37:49,338 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/B is initiating minor compaction (all files) 2024-11-10T04:37:49,338 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/A in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:49,338 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/B in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:49,338 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/87c09918febe428cbe01e1e5b1aed8b5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7f26a420f1ec4bc987617032e1152fc9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/86a9f758380a437680f518388c8fa26e] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=37.9 K 2024-11-10T04:37:49,338 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a86f16f465674f6eadb5374a26b9fd8a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d83ca5fc630b4fdc9044f9e36fe8f768, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e9780db6b48d42548d5f4c9975018c36] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=35.5 K 2024-11-10T04:37:49,338 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c09918febe428cbe01e1e5b1aed8b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213465923 2024-11-10T04:37:49,338 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a86f16f465674f6eadb5374a26b9fd8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213465923 2024-11-10T04:37:49,339 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d83ca5fc630b4fdc9044f9e36fe8f768, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731213466258 2024-11-10T04:37:49,339 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f26a420f1ec4bc987617032e1152fc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731213466258 2024-11-10T04:37:49,341 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e9780db6b48d42548d5f4c9975018c36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731213468073 2024-11-10T04:37:49,341 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a9f758380a437680f518388c8fa26e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731213466930 2024-11-10T04:37:49,364 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:49,364 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/756f97604fb04acc83485943ac3b310a is 50, key is test_row_0/A:col10/1731213468073/Put/seqid=0 2024-11-10T04:37:49,367 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:49,368 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/6b84d5fe93e04d8fa539d6b8d42f44a2 is 50, key is test_row_0/B:col10/1731213468073/Put/seqid=0 2024-11-10T04:37:49,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742070_1246 (size=12493) 2024-11-10T04:37:49,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T04:37:49,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:49,390 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-10T04:37:49,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742071_1247 (size=12493) 2024-11-10T04:37:49,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/237e7ef235a94e0d99ef75f10f5abec5 is 50, key is test_row_0/A:col10/1731213468079/Put/seqid=0 2024-11-10T04:37:49,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742072_1248 (size=9757) 2024-11-10T04:37:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T04:37:49,779 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/756f97604fb04acc83485943ac3b310a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/756f97604fb04acc83485943ac3b310a 2024-11-10T04:37:49,785 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/A of 305ea5ff5a8e2066f94db3068ceb2842 into 756f97604fb04acc83485943ac3b310a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:49,785 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:49,785 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/A, priority=13, startTime=1731213469336; duration=0sec 2024-11-10T04:37:49,785 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:49,785 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:A 2024-11-10T04:37:49,785 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:49,786 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:49,787 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/C is initiating minor compaction (all files) 2024-11-10T04:37:49,787 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/C in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:49,787 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5375759b318748eaaff9975d0d929b2a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/d3b335108ea34081883e381719b6b141, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/302c3bc1307847afb2c687f62560f299] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=35.5 K 2024-11-10T04:37:49,787 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5375759b318748eaaff9975d0d929b2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731213465923 2024-11-10T04:37:49,788 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b335108ea34081883e381719b6b141, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731213466258 2024-11-10T04:37:49,788 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 302c3bc1307847afb2c687f62560f299, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731213468073 2024-11-10T04:37:49,797 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#C#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:49,798 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/38274e240b284825b3ad8016bf801c7e is 50, key is test_row_0/C:col10/1731213468073/Put/seqid=0 2024-11-10T04:37:49,800 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/6b84d5fe93e04d8fa539d6b8d42f44a2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6b84d5fe93e04d8fa539d6b8d42f44a2 2024-11-10T04:37:49,804 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/237e7ef235a94e0d99ef75f10f5abec5 2024-11-10T04:37:49,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742073_1249 (size=12493) 2024-11-10T04:37:49,807 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/B of 305ea5ff5a8e2066f94db3068ceb2842 into 6b84d5fe93e04d8fa539d6b8d42f44a2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:49,807 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:49,807 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/B, priority=13, startTime=1731213469336; duration=0sec 2024-11-10T04:37:49,807 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:49,807 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:B 2024-11-10T04:37:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/fce5720ff13b414db0cb3eb91a9cc2fc is 50, key is test_row_0/B:col10/1731213468079/Put/seqid=0 2024-11-10T04:37:49,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742074_1250 (size=9757) 2024-11-10T04:37:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:50,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:50,211 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/38274e240b284825b3ad8016bf801c7e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/38274e240b284825b3ad8016bf801c7e 2024-11-10T04:37:50,216 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/C of 305ea5ff5a8e2066f94db3068ceb2842 into 38274e240b284825b3ad8016bf801c7e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:50,216 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:50,216 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/C, priority=13, startTime=1731213469336; duration=0sec 2024-11-10T04:37:50,217 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:50,217 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:C 2024-11-10T04:37:50,220 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/fce5720ff13b414db0cb3eb91a9cc2fc 2024-11-10T04:37:50,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213530223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:50,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2a31b9b636434dc68886a076a524c942 is 50, key is test_row_0/C:col10/1731213468079/Put/seqid=0 2024-11-10T04:37:50,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742075_1251 (size=9757) 2024-11-10T04:37:50,234 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2a31b9b636434dc68886a076a524c942 2024-11-10T04:37:50,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/237e7ef235a94e0d99ef75f10f5abec5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/237e7ef235a94e0d99ef75f10f5abec5 2024-11-10T04:37:50,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213530241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:50,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213530241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:50,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:50,244 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213530241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:50,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:50,245 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/237e7ef235a94e0d99ef75f10f5abec5, entries=100, sequenceid=165, filesize=9.5 K 2024-11-10T04:37:50,245 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:50,245 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:50,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/fce5720ff13b414db0cb3eb91a9cc2fc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fce5720ff13b414db0cb3eb91a9cc2fc 2024-11-10T04:37:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:50,251 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fce5720ff13b414db0cb3eb91a9cc2fc, entries=100, sequenceid=165, filesize=9.5 K 2024-11-10T04:37:50,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2a31b9b636434dc68886a076a524c942 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2a31b9b636434dc68886a076a524c942 2024-11-10T04:37:50,256 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2a31b9b636434dc68886a076a524c942, entries=100, sequenceid=165, filesize=9.5 K 2024-11-10T04:37:50,258 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 305ea5ff5a8e2066f94db3068ceb2842 in 868ms, sequenceid=165, compaction requested=false 2024-11-10T04:37:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:50,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-10T04:37:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-10T04:37:50,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T04:37:50,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6340 sec 2024-11-10T04:37:50,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6400 sec 2024-11-10T04:37:50,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:50,333 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:50,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-10T04:37:50,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/84178bf4232943eaafbf58f8d153ce43 is 50, key is test_row_0/A:col10/1731213470221/Put/seqid=0 2024-11-10T04:37:50,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213530342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:50,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742076_1252 (size=14541) 2024-11-10T04:37:50,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/84178bf4232943eaafbf58f8d153ce43 2024-11-10T04:37:50,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/fec2b2a30ea84975b465216235f9fe85 is 50, key is test_row_0/B:col10/1731213470221/Put/seqid=0 2024-11-10T04:37:50,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742077_1253 (size=12151) 2024-11-10T04:37:50,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/fec2b2a30ea84975b465216235f9fe85 2024-11-10T04:37:50,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/3cb1b8acb9b644558411b37d9d0c5575 is 50, key is test_row_0/C:col10/1731213470221/Put/seqid=0 2024-11-10T04:37:50,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742078_1254 (size=12151) 2024-11-10T04:37:50,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213530451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:50,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213530661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T04:37:50,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:50,772 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-10T04:37:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T04:37:50,774 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:50,774 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:50,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:50,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/3cb1b8acb9b644558411b37d9d0c5575 2024-11-10T04:37:50,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/84178bf4232943eaafbf58f8d153ce43 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/84178bf4232943eaafbf58f8d153ce43 2024-11-10T04:37:50,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/84178bf4232943eaafbf58f8d153ce43, entries=200, sequenceid=199, filesize=14.2 K 2024-11-10T04:37:50,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/fec2b2a30ea84975b465216235f9fe85 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fec2b2a30ea84975b465216235f9fe85 2024-11-10T04:37:50,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fec2b2a30ea84975b465216235f9fe85, entries=150, sequenceid=199, filesize=11.9 K 2024-11-10T04:37:50,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/3cb1b8acb9b644558411b37d9d0c5575 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/3cb1b8acb9b644558411b37d9d0c5575 2024-11-10T04:37:50,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/3cb1b8acb9b644558411b37d9d0c5575, entries=150, sequenceid=199, filesize=11.9 K 2024-11-10T04:37:50,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 305ea5ff5a8e2066f94db3068ceb2842 in 477ms, sequenceid=199, compaction requested=true 2024-11-10T04:37:50,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:50,810 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:50,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:50,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:50,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:50,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:50,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:50,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:50,811 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:50,812 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:50,812 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:50,812 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/B is initiating minor compaction (all files) 2024-11-10T04:37:50,812 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/B in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:50,812 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6b84d5fe93e04d8fa539d6b8d42f44a2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fce5720ff13b414db0cb3eb91a9cc2fc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fec2b2a30ea84975b465216235f9fe85] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=33.6 K 2024-11-10T04:37:50,813 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/A is initiating minor compaction (all files) 2024-11-10T04:37:50,813 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b84d5fe93e04d8fa539d6b8d42f44a2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731213468073 2024-11-10T04:37:50,813 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/A in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:50,813 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/756f97604fb04acc83485943ac3b310a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/237e7ef235a94e0d99ef75f10f5abec5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/84178bf4232943eaafbf58f8d153ce43] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=35.9 K 2024-11-10T04:37:50,813 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fce5720ff13b414db0cb3eb91a9cc2fc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731213468078 2024-11-10T04:37:50,814 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fec2b2a30ea84975b465216235f9fe85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731213470211 2024-11-10T04:37:50,814 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 756f97604fb04acc83485943ac3b310a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731213468073 2024-11-10T04:37:50,814 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 237e7ef235a94e0d99ef75f10f5abec5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731213468078 2024-11-10T04:37:50,815 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84178bf4232943eaafbf58f8d153ce43, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731213470201 2024-11-10T04:37:50,822 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#A#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:50,822 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:50,823 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/7ad927e2c9b540e1b2101e51efba67b5 is 50, key is test_row_0/A:col10/1731213470221/Put/seqid=0 2024-11-10T04:37:50,823 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/ca70fd5b6f9747eab523cf862ca425df is 50, key is test_row_0/B:col10/1731213470221/Put/seqid=0 2024-11-10T04:37:50,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742079_1255 (size=12595) 2024-11-10T04:37:50,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742080_1256 (size=12595) 2024-11-10T04:37:50,834 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/7ad927e2c9b540e1b2101e51efba67b5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7ad927e2c9b540e1b2101e51efba67b5 2024-11-10T04:37:50,840 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/A of 305ea5ff5a8e2066f94db3068ceb2842 into 7ad927e2c9b540e1b2101e51efba67b5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:50,840 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:50,840 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/A, priority=13, startTime=1731213470810; duration=0sec 2024-11-10T04:37:50,840 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:50,841 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:A 2024-11-10T04:37:50,841 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:50,842 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:50,842 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/C is initiating minor compaction (all files) 2024-11-10T04:37:50,842 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/C in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:50,842 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/38274e240b284825b3ad8016bf801c7e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2a31b9b636434dc68886a076a524c942, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/3cb1b8acb9b644558411b37d9d0c5575] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=33.6 K 2024-11-10T04:37:50,843 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38274e240b284825b3ad8016bf801c7e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731213468073 2024-11-10T04:37:50,843 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a31b9b636434dc68886a076a524c942, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731213468078 2024-11-10T04:37:50,844 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cb1b8acb9b644558411b37d9d0c5575, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731213470211 2024-11-10T04:37:50,851 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:50,852 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/e0dd3aa2452842a6907f8b9c0453c4c9 is 50, key is test_row_0/C:col10/1731213470221/Put/seqid=0 2024-11-10T04:37:50,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742081_1257 (size=12595) 2024-11-10T04:37:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T04:37:50,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T04:37:50,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:50,928 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-10T04:37:50,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/ae28bee4094c487e9f7308bc54aec2aa is 50, key is test_row_0/A:col10/1731213470337/Put/seqid=0 2024-11-10T04:37:50,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742082_1258 (size=9757) 2024-11-10T04:37:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:50,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:51,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213531078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T04:37:51,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213531111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:51,114 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213531191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:51,235 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/ca70fd5b6f9747eab523cf862ca425df as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/ca70fd5b6f9747eab523cf862ca425df 2024-11-10T04:37:51,240 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/B of 305ea5ff5a8e2066f94db3068ceb2842 into ca70fd5b6f9747eab523cf862ca425df(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:51,240 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:51,240 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/B, priority=13, startTime=1731213470811; duration=0sec 2024-11-10T04:37:51,240 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:51,240 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:B 2024-11-10T04:37:51,263 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/e0dd3aa2452842a6907f8b9c0453c4c9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e0dd3aa2452842a6907f8b9c0453c4c9 2024-11-10T04:37:51,268 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/C of 305ea5ff5a8e2066f94db3068ceb2842 into e0dd3aa2452842a6907f8b9c0453c4c9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:51,268 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:51,268 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/C, priority=13, startTime=1731213470811; duration=0sec 2024-11-10T04:37:51,268 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:51,268 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:C 2024-11-10T04:37:51,349 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/ae28bee4094c487e9f7308bc54aec2aa 2024-11-10T04:37:51,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/f74fa30ffcf34f46bd0a441f6d582c92 is 50, key is test_row_0/B:col10/1731213470337/Put/seqid=0 2024-11-10T04:37:51,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742083_1259 (size=9757) 2024-11-10T04:37:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T04:37:51,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213531401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:51,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213531711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:51,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:51,781 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/f74fa30ffcf34f46bd0a441f6d582c92 2024-11-10T04:37:51,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/0deb31e7cad247dba8ffef2229029ff4 is 50, key is test_row_0/C:col10/1731213470337/Put/seqid=0 2024-11-10T04:37:51,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742084_1260 (size=9757) 2024-11-10T04:37:51,802 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/0deb31e7cad247dba8ffef2229029ff4 2024-11-10T04:37:51,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/ae28bee4094c487e9f7308bc54aec2aa as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/ae28bee4094c487e9f7308bc54aec2aa 2024-11-10T04:37:51,811 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/ae28bee4094c487e9f7308bc54aec2aa, entries=100, sequenceid=205, filesize=9.5 K 2024-11-10T04:37:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/f74fa30ffcf34f46bd0a441f6d582c92 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/f74fa30ffcf34f46bd0a441f6d582c92 2024-11-10T04:37:51,817 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/f74fa30ffcf34f46bd0a441f6d582c92, entries=100, sequenceid=205, filesize=9.5 K 2024-11-10T04:37:51,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/0deb31e7cad247dba8ffef2229029ff4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0deb31e7cad247dba8ffef2229029ff4 2024-11-10T04:37:51,822 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0deb31e7cad247dba8ffef2229029ff4, entries=100, sequenceid=205, filesize=9.5 K 2024-11-10T04:37:51,823 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=187.85 KB/192360 for 305ea5ff5a8e2066f94db3068ceb2842 in 896ms, sequenceid=205, compaction requested=false 2024-11-10T04:37:51,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:51,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:51,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-10T04:37:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-10T04:37:51,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-10T04:37:51,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0510 sec 2024-11-10T04:37:51,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0600 sec 2024-11-10T04:37:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T04:37:51,911 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:51,912 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-10T04:37:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T04:37:51,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:51,915 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:51,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T04:37:52,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T04:37:52,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:52,068 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-10T04:37:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/88d671dbb8c442028363eca8c7f9e702 is 50, key is test_row_0/A:col10/1731213471071/Put/seqid=0 2024-11-10T04:37:52,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742085_1261 (size=12151) 2024-11-10T04:37:52,086 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/88d671dbb8c442028363eca8c7f9e702 2024-11-10T04:37:52,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/c3d4a2ae35f34dc98f3f11b25b04fc07 is 50, key is test_row_0/B:col10/1731213471071/Put/seqid=0 2024-11-10T04:37:52,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742086_1262 (size=12151) 2024-11-10T04:37:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:52,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:52,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213532227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T04:37:52,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213532251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:52,252 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:52,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213532271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:52,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213532271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:52,273 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:52,273 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:52,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213532341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:52,501 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/c3d4a2ae35f34dc98f3f11b25b04fc07 2024-11-10T04:37:52,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/b4f51dafe6184bddad90ce4d4a6cac01 is 50, key is test_row_0/C:col10/1731213471071/Put/seqid=0 2024-11-10T04:37:52,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742087_1263 (size=12151) 2024-11-10T04:37:52,515 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/b4f51dafe6184bddad90ce4d4a6cac01 2024-11-10T04:37:52,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/88d671dbb8c442028363eca8c7f9e702 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/88d671dbb8c442028363eca8c7f9e702 2024-11-10T04:37:52,524 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/88d671dbb8c442028363eca8c7f9e702, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T04:37:52,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/c3d4a2ae35f34dc98f3f11b25b04fc07 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/c3d4a2ae35f34dc98f3f11b25b04fc07 2024-11-10T04:37:52,530 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/c3d4a2ae35f34dc98f3f11b25b04fc07, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T04:37:52,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/b4f51dafe6184bddad90ce4d4a6cac01 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b4f51dafe6184bddad90ce4d4a6cac01 2024-11-10T04:37:52,535 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b4f51dafe6184bddad90ce4d4a6cac01, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T04:37:52,536 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 305ea5ff5a8e2066f94db3068ceb2842 in 469ms, sequenceid=238, compaction requested=true 2024-11-10T04:37:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:52,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-10T04:37:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-10T04:37:52,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-10T04:37:52,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 622 msec 2024-11-10T04:37:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T04:37:52,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 628 msec 2024-11-10T04:37:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:52,566 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:52,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:37:52,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/e426929e1a504df28723e72127f99c44 is 50, key is test_row_0/A:col10/1731213472557/Put/seqid=0 2024-11-10T04:37:52,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742088_1264 (size=14541) 2024-11-10T04:37:52,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213532694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:52,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213532801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:52,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/e426929e1a504df28723e72127f99c44 2024-11-10T04:37:52,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/7ed362ed8d0641a8bcc39ba6c66c2bdb is 50, key is test_row_0/B:col10/1731213472557/Put/seqid=0 2024-11-10T04:37:52,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742089_1265 (size=12151) 2024-11-10T04:37:53,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213533011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T04:37:53,051 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:53,051 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-10T04:37:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T04:37:53,053 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:53,054 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:53,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T04:37:53,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T04:37:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213533321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:53,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T04:37:53,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:53,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T04:37:53,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/7ed362ed8d0641a8bcc39ba6c66c2bdb 2024-11-10T04:37:53,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/910adde0118143328852673c2ce5c829 is 50, key is test_row_0/C:col10/1731213472557/Put/seqid=0 2024-11-10T04:37:53,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742090_1266 (size=12151) 2024-11-10T04:37:53,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T04:37:53,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:53,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T04:37:53,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T04:37:53,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/910adde0118143328852673c2ce5c829 2024-11-10T04:37:53,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/e426929e1a504df28723e72127f99c44 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/e426929e1a504df28723e72127f99c44 2024-11-10T04:37:53,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/e426929e1a504df28723e72127f99c44, entries=200, sequenceid=249, filesize=14.2 K 2024-11-10T04:37:53,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/7ed362ed8d0641a8bcc39ba6c66c2bdb as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/7ed362ed8d0641a8bcc39ba6c66c2bdb 2024-11-10T04:37:53,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/7ed362ed8d0641a8bcc39ba6c66c2bdb, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T04:37:53,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/910adde0118143328852673c2ce5c829 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/910adde0118143328852673c2ce5c829 2024-11-10T04:37:53,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T04:37:53,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:53,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/910adde0118143328852673c2ce5c829, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T04:37:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 305ea5ff5a8e2066f94db3068ceb2842 in 1255ms, sequenceid=249, compaction requested=true 2024-11-10T04:37:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:53,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:53,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:53,822 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:53,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:53,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:53,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:53,822 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:53,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:53,823 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49044 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:53,823 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/A is initiating minor compaction (all files) 2024-11-10T04:37:53,823 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:53,823 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/B is initiating minor compaction (all files) 2024-11-10T04:37:53,823 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/A in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,823 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/B in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,823 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7ad927e2c9b540e1b2101e51efba67b5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/ae28bee4094c487e9f7308bc54aec2aa, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/88d671dbb8c442028363eca8c7f9e702, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/e426929e1a504df28723e72127f99c44] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=47.9 K 2024-11-10T04:37:53,823 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/ca70fd5b6f9747eab523cf862ca425df, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/f74fa30ffcf34f46bd0a441f6d582c92, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/c3d4a2ae35f34dc98f3f11b25b04fc07, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/7ed362ed8d0641a8bcc39ba6c66c2bdb] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=45.6 K 2024-11-10T04:37:53,823 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad927e2c9b540e1b2101e51efba67b5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731213470211 2024-11-10T04:37:53,823 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ca70fd5b6f9747eab523cf862ca425df, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731213470211 2024-11-10T04:37:53,824 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae28bee4094c487e9f7308bc54aec2aa, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731213470337 2024-11-10T04:37:53,824 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f74fa30ffcf34f46bd0a441f6d582c92, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731213470337 2024-11-10T04:37:53,825 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c3d4a2ae35f34dc98f3f11b25b04fc07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731213471032 2024-11-10T04:37:53,825 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d671dbb8c442028363eca8c7f9e702, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731213471032 2024-11-10T04:37:53,825 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed362ed8d0641a8bcc39ba6c66c2bdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213472557 2024-11-10T04:37:53,825 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e426929e1a504df28723e72127f99c44, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213472556 2024-11-10T04:37:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:53,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:37:53,835 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:53,835 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#A#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:53,836 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/4a35ee88388446d1985068da085be651 is 50, key is test_row_0/B:col10/1731213472557/Put/seqid=0 2024-11-10T04:37:53,836 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/54f0288a00894d9b85cebb0138057de6 is 50, key is test_row_0/A:col10/1731213472557/Put/seqid=0 2024-11-10T04:37:53,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/27da507eb78d4651a0e3ec2b2ab9e104 is 50, key is test_row_0/A:col10/1731213472692/Put/seqid=0 2024-11-10T04:37:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742091_1267 (size=12731) 2024-11-10T04:37:53,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742092_1268 (size=14741) 2024-11-10T04:37:53,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/27da507eb78d4651a0e3ec2b2ab9e104 2024-11-10T04:37:53,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742093_1269 (size=12731) 2024-11-10T04:37:53,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/e6e1178780c6473883cef4c9745427e5 is 50, key is test_row_0/B:col10/1731213472692/Put/seqid=0 2024-11-10T04:37:53,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742094_1270 (size=12301) 2024-11-10T04:37:53,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213533870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:53,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T04:37:53,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:53,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:53,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213533981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:54,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T04:37:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:54,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:54,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:54,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:54,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T04:37:54,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213534192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:54,252 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/4a35ee88388446d1985068da085be651 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a35ee88388446d1985068da085be651 2024-11-10T04:37:54,257 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/B of 305ea5ff5a8e2066f94db3068ceb2842 into 4a35ee88388446d1985068da085be651(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:54,257 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:54,257 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/B, priority=12, startTime=1731213473822; duration=0sec 2024-11-10T04:37:54,257 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:54,257 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:B 2024-11-10T04:37:54,257 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:54,259 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:54,259 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/C is initiating minor compaction (all files) 2024-11-10T04:37:54,259 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/C in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:54,259 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e0dd3aa2452842a6907f8b9c0453c4c9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0deb31e7cad247dba8ffef2229029ff4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b4f51dafe6184bddad90ce4d4a6cac01, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/910adde0118143328852673c2ce5c829] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=45.6 K 2024-11-10T04:37:54,259 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e0dd3aa2452842a6907f8b9c0453c4c9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731213470211 2024-11-10T04:37:54,260 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0deb31e7cad247dba8ffef2229029ff4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731213470337 2024-11-10T04:37:54,260 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b4f51dafe6184bddad90ce4d4a6cac01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731213471032 2024-11-10T04:37:54,260 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 910adde0118143328852673c2ce5c829, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213472557 2024-11-10T04:37:54,266 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/54f0288a00894d9b85cebb0138057de6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/54f0288a00894d9b85cebb0138057de6 2024-11-10T04:37:54,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/e6e1178780c6473883cef4c9745427e5 2024-11-10T04:37:54,272 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/A of 305ea5ff5a8e2066f94db3068ceb2842 into 54f0288a00894d9b85cebb0138057de6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:54,273 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:54,273 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/A, priority=12, startTime=1731213473821; duration=0sec 2024-11-10T04:37:54,273 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:54,273 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:A 2024-11-10T04:37:54,273 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#C#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:54,274 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/8ae18659cb7c44deafdbe8ebb6e20d08 is 50, key is test_row_0/C:col10/1731213472557/Put/seqid=0 2024-11-10T04:37:54,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/e4a7d5e781154a9bb4e2a9e608d6e52a is 50, key is test_row_0/C:col10/1731213472692/Put/seqid=0 2024-11-10T04:37:54,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T04:37:54,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:54,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:54,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:54,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:54,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:54,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742095_1271 (size=12731) 2024-11-10T04:37:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:54,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742096_1272 (size=12301) 2024-11-10T04:37:54,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/e4a7d5e781154a9bb4e2a9e608d6e52a 2024-11-10T04:37:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/27da507eb78d4651a0e3ec2b2ab9e104 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/27da507eb78d4651a0e3ec2b2ab9e104 2024-11-10T04:37:54,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/27da507eb78d4651a0e3ec2b2ab9e104, entries=200, sequenceid=275, filesize=14.4 K 2024-11-10T04:37:54,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/e6e1178780c6473883cef4c9745427e5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e6e1178780c6473883cef4c9745427e5 2024-11-10T04:37:54,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e6e1178780c6473883cef4c9745427e5, entries=150, sequenceid=275, filesize=12.0 K 2024-11-10T04:37:54,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/e4a7d5e781154a9bb4e2a9e608d6e52a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e4a7d5e781154a9bb4e2a9e608d6e52a 2024-11-10T04:37:54,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e4a7d5e781154a9bb4e2a9e608d6e52a, entries=150, sequenceid=275, filesize=12.0 K 2024-11-10T04:37:54,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 305ea5ff5a8e2066f94db3068ceb2842 in 471ms, sequenceid=275, compaction requested=false 2024-11-10T04:37:54,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:54,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T04:37:54,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:54,435 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T04:37:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/b829af1a6fb24b82a806df560b0c91f5 is 50, key is test_row_0/A:col10/1731213473869/Put/seqid=0 2024-11-10T04:37:54,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742097_1273 (size=9857) 2024-11-10T04:37:54,448 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/b829af1a6fb24b82a806df560b0c91f5 2024-11-10T04:37:54,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/cfb63325184349a79df52ac49b643d28 is 50, key is test_row_0/B:col10/1731213473869/Put/seqid=0 2024-11-10T04:37:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742098_1274 (size=9857) 2024-11-10T04:37:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:54,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:54,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213534602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:54,688 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/8ae18659cb7c44deafdbe8ebb6e20d08 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8ae18659cb7c44deafdbe8ebb6e20d08 2024-11-10T04:37:54,694 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/C of 305ea5ff5a8e2066f94db3068ceb2842 into 8ae18659cb7c44deafdbe8ebb6e20d08(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:54,694 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:54,694 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/C, priority=12, startTime=1731213473822; duration=0sec 2024-11-10T04:37:54,694 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:54,694 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:C 2024-11-10T04:37:54,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213534711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:54,870 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/cfb63325184349a79df52ac49b643d28 2024-11-10T04:37:54,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/6750b9da36dc4b09ac69d72fa75ea0db is 50, key is test_row_0/C:col10/1731213473869/Put/seqid=0 2024-11-10T04:37:54,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742099_1275 (size=9857) 2024-11-10T04:37:54,881 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/6750b9da36dc4b09ac69d72fa75ea0db 2024-11-10T04:37:54,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/b829af1a6fb24b82a806df560b0c91f5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b829af1a6fb24b82a806df560b0c91f5 2024-11-10T04:37:54,890 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b829af1a6fb24b82a806df560b0c91f5, entries=100, sequenceid=287, filesize=9.6 K 2024-11-10T04:37:54,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/cfb63325184349a79df52ac49b643d28 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/cfb63325184349a79df52ac49b643d28 2024-11-10T04:37:54,894 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/cfb63325184349a79df52ac49b643d28, entries=100, sequenceid=287, filesize=9.6 K 2024-11-10T04:37:54,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/6750b9da36dc4b09ac69d72fa75ea0db as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6750b9da36dc4b09ac69d72fa75ea0db 2024-11-10T04:37:54,899 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6750b9da36dc4b09ac69d72fa75ea0db, entries=100, sequenceid=287, filesize=9.6 K 2024-11-10T04:37:54,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 305ea5ff5a8e2066f94db3068ceb2842 in 465ms, sequenceid=287, compaction requested=true 2024-11-10T04:37:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-10T04:37:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-10T04:37:54,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T04:37:54,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8480 sec 2024-11-10T04:37:54,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8530 sec 2024-11-10T04:37:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:54,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:54,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T04:37:54,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/110854ef0bd44e2aa511f9400d22fc2f is 50, key is test_row_0/A:col10/1731213474601/Put/seqid=0 2024-11-10T04:37:54,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742100_1276 (size=14741) 2024-11-10T04:37:54,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/110854ef0bd44e2aa511f9400d22fc2f 2024-11-10T04:37:54,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/af98b641c93744fa9bb56883736f2eb7 is 50, key is test_row_0/B:col10/1731213474601/Put/seqid=0 2024-11-10T04:37:54,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742101_1277 (size=12301) 2024-11-10T04:37:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213534949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213535061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:55,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48174 deadline: 1731213535161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:55,164 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T04:37:55,201 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:55,201 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-10T04:37:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T04:37:55,203 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:55,204 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:55,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:55,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213535271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T04:37:55,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/af98b641c93744fa9bb56883736f2eb7 2024-11-10T04:37:55,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/5a1673914aaa4df39b5ea4d6cbaa86e2 is 50, key is test_row_0/C:col10/1731213474601/Put/seqid=0 2024-11-10T04:37:55,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T04:37:55,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:55,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:55,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:55,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:55,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:55,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742102_1278 (size=12301) 2024-11-10T04:37:55,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T04:37:55,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:55,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:55,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:55,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T04:37:55,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213535581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:55,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T04:37:55,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:55,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:55,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:55,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:55,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:55,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/5a1673914aaa4df39b5ea4d6cbaa86e2 2024-11-10T04:37:55,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/110854ef0bd44e2aa511f9400d22fc2f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/110854ef0bd44e2aa511f9400d22fc2f 2024-11-10T04:37:55,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/110854ef0bd44e2aa511f9400d22fc2f, entries=200, sequenceid=315, filesize=14.4 K 2024-11-10T04:37:55,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/af98b641c93744fa9bb56883736f2eb7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/af98b641c93744fa9bb56883736f2eb7 2024-11-10T04:37:55,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/af98b641c93744fa9bb56883736f2eb7, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T04:37:55,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/5a1673914aaa4df39b5ea4d6cbaa86e2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5a1673914aaa4df39b5ea4d6cbaa86e2 2024-11-10T04:37:55,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5a1673914aaa4df39b5ea4d6cbaa86e2, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T04:37:55,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 305ea5ff5a8e2066f94db3068ceb2842 in 857ms, sequenceid=315, compaction requested=true 2024-11-10T04:37:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:55,781 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:55,781 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:55,783 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:55,783 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/B is initiating minor compaction (all files) 2024-11-10T04:37:55,783 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52070 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:55,783 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/A is initiating minor compaction (all files) 2024-11-10T04:37:55,783 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/B in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:55,783 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/A in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:55,783 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a35ee88388446d1985068da085be651, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e6e1178780c6473883cef4c9745427e5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/cfb63325184349a79df52ac49b643d28, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/af98b641c93744fa9bb56883736f2eb7] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=46.1 K 2024-11-10T04:37:55,783 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/54f0288a00894d9b85cebb0138057de6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/27da507eb78d4651a0e3ec2b2ab9e104, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b829af1a6fb24b82a806df560b0c91f5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/110854ef0bd44e2aa511f9400d22fc2f] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=50.8 K 2024-11-10T04:37:55,783 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a35ee88388446d1985068da085be651, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213472557 2024-11-10T04:37:55,784 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f0288a00894d9b85cebb0138057de6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213472557 2024-11-10T04:37:55,784 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e1178780c6473883cef4c9745427e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731213472686 2024-11-10T04:37:55,784 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27da507eb78d4651a0e3ec2b2ab9e104, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731213472674 2024-11-10T04:37:55,784 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb63325184349a79df52ac49b643d28, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731213473843 2024-11-10T04:37:55,784 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b829af1a6fb24b82a806df560b0c91f5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731213473843 2024-11-10T04:37:55,785 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting af98b641c93744fa9bb56883736f2eb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731213474586 2024-11-10T04:37:55,785 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 110854ef0bd44e2aa511f9400d22fc2f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731213474586 2024-11-10T04:37:55,794 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#B#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:55,795 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/a7c8c93e72cd410a9a180a8e86779ca7 is 50, key is test_row_0/B:col10/1731213474601/Put/seqid=0 2024-11-10T04:37:55,797 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#A#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:55,798 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/c56d6e7d47ec4c9aad9fbf5e4720f164 is 50, key is test_row_0/A:col10/1731213474601/Put/seqid=0 2024-11-10T04:37:55,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742103_1279 (size=13017) 2024-11-10T04:37:55,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T04:37:55,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:55,816 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-10T04:37:55,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742104_1280 (size=13017) 2024-11-10T04:37:55,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/1355acc051f24eec9f439b5b1f96e049 is 50, key is test_row_0/A:col10/1731213474942/Put/seqid=0 2024-11-10T04:37:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T04:37:55,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742105_1281 (size=12301) 2024-11-10T04:37:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:56,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:56,208 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/a7c8c93e72cd410a9a180a8e86779ca7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a7c8c93e72cd410a9a180a8e86779ca7 2024-11-10T04:37:56,217 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/B of 305ea5ff5a8e2066f94db3068ceb2842 into a7c8c93e72cd410a9a180a8e86779ca7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:56,217 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:56,218 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/B, priority=12, startTime=1731213475781; duration=0sec 2024-11-10T04:37:56,218 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:56,218 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:B 2024-11-10T04:37:56,218 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:37:56,219 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:37:56,219 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/C is initiating minor compaction (all files) 2024-11-10T04:37:56,220 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/C in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,220 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8ae18659cb7c44deafdbe8ebb6e20d08, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e4a7d5e781154a9bb4e2a9e608d6e52a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6750b9da36dc4b09ac69d72fa75ea0db, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5a1673914aaa4df39b5ea4d6cbaa86e2] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=46.1 K 2024-11-10T04:37:56,220 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae18659cb7c44deafdbe8ebb6e20d08, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213472557 2024-11-10T04:37:56,221 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e4a7d5e781154a9bb4e2a9e608d6e52a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731213472686 2024-11-10T04:37:56,221 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6750b9da36dc4b09ac69d72fa75ea0db, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731213473843 2024-11-10T04:37:56,222 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a1673914aaa4df39b5ea4d6cbaa86e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731213474586 2024-11-10T04:37:56,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213536218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:56,226 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/c56d6e7d47ec4c9aad9fbf5e4720f164 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/c56d6e7d47ec4c9aad9fbf5e4720f164 2024-11-10T04:37:56,232 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/A of 305ea5ff5a8e2066f94db3068ceb2842 into c56d6e7d47ec4c9aad9fbf5e4720f164(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:56,232 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:56,232 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/A, priority=12, startTime=1731213475781; duration=0sec 2024-11-10T04:37:56,232 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:56,232 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:A 2024-11-10T04:37:56,237 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#C#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:56,238 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/1355acc051f24eec9f439b5b1f96e049 2024-11-10T04:37:56,241 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/567ca2302dca4a998d4d75511fc3144f is 50, key is test_row_0/C:col10/1731213474601/Put/seqid=0 2024-11-10T04:37:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/711f5724685d4ae79f3f76fd0dd30b51 is 50, key is test_row_0/B:col10/1731213474942/Put/seqid=0 2024-11-10T04:37:56,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742106_1282 (size=13017) 2024-11-10T04:37:56,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742107_1283 (size=12301) 2024-11-10T04:37:56,261 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/711f5724685d4ae79f3f76fd0dd30b51 2024-11-10T04:37:56,263 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/567ca2302dca4a998d4d75511fc3144f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/567ca2302dca4a998d4d75511fc3144f 2024-11-10T04:37:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/e140a68031814933902398c0fa0f8f20 is 50, key is test_row_0/C:col10/1731213474942/Put/seqid=0 2024-11-10T04:37:56,271 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/C of 305ea5ff5a8e2066f94db3068ceb2842 into 567ca2302dca4a998d4d75511fc3144f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:56,271 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:56,271 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/C, priority=12, startTime=1731213475781; duration=0sec 2024-11-10T04:37:56,271 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:56,271 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:C 2024-11-10T04:37:56,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48192 deadline: 1731213536281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:56,283 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:56,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742108_1284 (size=12301) 2024-11-10T04:37:56,290 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/e140a68031814933902398c0fa0f8f20 2024-11-10T04:37:56,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48178 deadline: 1731213536291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:56,293 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/1355acc051f24eec9f439b5b1f96e049 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/1355acc051f24eec9f439b5b1f96e049 2024-11-10T04:37:56,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48204 deadline: 1731213536302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:56,303 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 305ea5ff5a8e2066f94db3068ceb2842 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:37:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:56,304 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/1355acc051f24eec9f439b5b1f96e049, entries=150, sequenceid=324, filesize=12.0 K 2024-11-10T04:37:56,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/711f5724685d4ae79f3f76fd0dd30b51 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/711f5724685d4ae79f3f76fd0dd30b51 2024-11-10T04:37:56,310 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/711f5724685d4ae79f3f76fd0dd30b51, entries=150, sequenceid=324, filesize=12.0 K 2024-11-10T04:37:56,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/e140a68031814933902398c0fa0f8f20 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e140a68031814933902398c0fa0f8f20 2024-11-10T04:37:56,316 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e140a68031814933902398c0fa0f8f20, entries=150, sequenceid=324, filesize=12.0 K 2024-11-10T04:37:56,317 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 305ea5ff5a8e2066f94db3068ceb2842 in 501ms, sequenceid=324, compaction requested=false 2024-11-10T04:37:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-10T04:37:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-10T04:37:56,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T04:37:56,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1150 sec 2024-11-10T04:37:56,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.1210 sec 2024-11-10T04:37:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:56,333 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:56,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-10T04:37:56,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/05b938b0a8d14c619c6f5320c92fe3d9 is 50, key is test_row_0/A:col10/1731213476161/Put/seqid=0 2024-11-10T04:37:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T04:37:56,341 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:56,342 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-10T04:37:56,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T04:37:56,345 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:56,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:56,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742109_1285 (size=14741) 2024-11-10T04:37:56,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/05b938b0a8d14c619c6f5320c92fe3d9 2024-11-10T04:37:56,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213536357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:56,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/8d2a71284faf4c8baae072194743dce2 is 50, key is test_row_0/B:col10/1731213476161/Put/seqid=0 2024-11-10T04:37:56,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742110_1286 (size=12301) 2024-11-10T04:37:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T04:37:56,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213536471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:56,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T04:37:56,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:56,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T04:37:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T04:37:56,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213536681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:56,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/8d2a71284faf4c8baae072194743dce2 2024-11-10T04:37:56,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/88005617f7b641148fdf6537395a389b is 50, key is test_row_0/C:col10/1731213476161/Put/seqid=0 2024-11-10T04:37:56,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742111_1287 (size=12301) 2024-11-10T04:37:56,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T04:37:56,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:56,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T04:37:56,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:56,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:56,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T04:37:56,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213536991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:57,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T04:37:57,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:57,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/88005617f7b641148fdf6537395a389b 2024-11-10T04:37:57,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/05b938b0a8d14c619c6f5320c92fe3d9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/05b938b0a8d14c619c6f5320c92fe3d9 2024-11-10T04:37:57,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/05b938b0a8d14c619c6f5320c92fe3d9, entries=200, sequenceid=355, filesize=14.4 K 2024-11-10T04:37:57,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/8d2a71284faf4c8baae072194743dce2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8d2a71284faf4c8baae072194743dce2 2024-11-10T04:37:57,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8d2a71284faf4c8baae072194743dce2, entries=150, sequenceid=355, filesize=12.0 K 2024-11-10T04:37:57,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/88005617f7b641148fdf6537395a389b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/88005617f7b641148fdf6537395a389b 2024-11-10T04:37:57,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/88005617f7b641148fdf6537395a389b, entries=150, sequenceid=355, filesize=12.0 K 2024-11-10T04:37:57,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 305ea5ff5a8e2066f94db3068ceb2842 in 870ms, sequenceid=355, compaction requested=true 2024-11-10T04:37:57,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:57,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:57,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:57,203 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:57,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:57,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:57,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:57,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:57,203 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:57,204 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:57,204 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:57,204 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/B is initiating minor compaction (all files) 2024-11-10T04:37:57,204 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/A is initiating minor compaction (all files) 2024-11-10T04:37:57,205 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/B in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,205 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/A in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,205 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a7c8c93e72cd410a9a180a8e86779ca7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/711f5724685d4ae79f3f76fd0dd30b51, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8d2a71284faf4c8baae072194743dce2] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=36.7 K 2024-11-10T04:37:57,205 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/c56d6e7d47ec4c9aad9fbf5e4720f164, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/1355acc051f24eec9f439b5b1f96e049, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/05b938b0a8d14c619c6f5320c92fe3d9] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=39.1 K 2024-11-10T04:37:57,205 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c56d6e7d47ec4c9aad9fbf5e4720f164, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731213474586 2024-11-10T04:37:57,205 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c8c93e72cd410a9a180a8e86779ca7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731213474586 2024-11-10T04:37:57,205 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1355acc051f24eec9f439b5b1f96e049, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731213474935 2024-11-10T04:37:57,206 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 711f5724685d4ae79f3f76fd0dd30b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731213474935 2024-11-10T04:37:57,206 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05b938b0a8d14c619c6f5320c92fe3d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731213476161 2024-11-10T04:37:57,206 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2a71284faf4c8baae072194743dce2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731213476161 2024-11-10T04:37:57,213 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:57,214 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:57,214 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/6864d00395ff4a99915e854e9dc1107c is 50, key is test_row_0/A:col10/1731213476161/Put/seqid=0 2024-11-10T04:37:57,214 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/a1428ed3541f46ecb6320cc76507698b is 50, key is test_row_0/B:col10/1731213476161/Put/seqid=0 2024-11-10T04:37:57,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742113_1289 (size=13119) 2024-11-10T04:37:57,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742112_1288 (size=13119) 2024-11-10T04:37:57,226 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/a1428ed3541f46ecb6320cc76507698b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a1428ed3541f46ecb6320cc76507698b 2024-11-10T04:37:57,236 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/B of 305ea5ff5a8e2066f94db3068ceb2842 into a1428ed3541f46ecb6320cc76507698b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:57,237 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:57,237 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/B, priority=13, startTime=1731213477203; duration=0sec 2024-11-10T04:37:57,237 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:57,237 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:B 2024-11-10T04:37:57,237 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:57,238 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:57,238 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/C is initiating minor compaction (all files) 2024-11-10T04:37:57,238 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/C in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,238 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/567ca2302dca4a998d4d75511fc3144f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e140a68031814933902398c0fa0f8f20, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/88005617f7b641148fdf6537395a389b] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=36.7 K 2024-11-10T04:37:57,239 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 567ca2302dca4a998d4d75511fc3144f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731213474586 2024-11-10T04:37:57,239 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e140a68031814933902398c0fa0f8f20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731213474935 2024-11-10T04:37:57,239 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 88005617f7b641148fdf6537395a389b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731213476161 2024-11-10T04:37:57,247 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:57,248 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/da0cb9269a2c42d3aa20f51413841db3 is 50, key is test_row_0/C:col10/1731213476161/Put/seqid=0 2024-11-10T04:37:57,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742114_1290 (size=13119) 2024-11-10T04:37:57,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T04:37:57,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,268 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T04:37:57,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/b0322dea124f46bca4f642bd694b52f5 is 50, key is test_row_0/A:col10/1731213476341/Put/seqid=0 2024-11-10T04:37:57,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742115_1291 (size=12301) 2024-11-10T04:37:57,279 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/b0322dea124f46bca4f642bd694b52f5 2024-11-10T04:37:57,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/4a5b63d0de684e12bc53546afe13a2dc is 50, key is test_row_0/B:col10/1731213476341/Put/seqid=0 2024-11-10T04:37:57,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742116_1292 (size=12301) 2024-11-10T04:37:57,295 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/4a5b63d0de684e12bc53546afe13a2dc 2024-11-10T04:37:57,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/81b250f67fd64eb79dcd5069c4b01c37 is 50, key is test_row_0/C:col10/1731213476341/Put/seqid=0 2024-11-10T04:37:57,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742117_1293 (size=12301) 2024-11-10T04:37:57,306 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/81b250f67fd64eb79dcd5069c4b01c37 2024-11-10T04:37:57,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/b0322dea124f46bca4f642bd694b52f5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b0322dea124f46bca4f642bd694b52f5 2024-11-10T04:37:57,314 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b0322dea124f46bca4f642bd694b52f5, entries=150, sequenceid=364, filesize=12.0 K 2024-11-10T04:37:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/4a5b63d0de684e12bc53546afe13a2dc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a5b63d0de684e12bc53546afe13a2dc 2024-11-10T04:37:57,319 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a5b63d0de684e12bc53546afe13a2dc, entries=150, sequenceid=364, filesize=12.0 K 2024-11-10T04:37:57,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/81b250f67fd64eb79dcd5069c4b01c37 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/81b250f67fd64eb79dcd5069c4b01c37 2024-11-10T04:37:57,325 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/81b250f67fd64eb79dcd5069c4b01c37, entries=150, sequenceid=364, filesize=12.0 K 2024-11-10T04:37:57,326 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 305ea5ff5a8e2066f94db3068ceb2842 in 58ms, sequenceid=364, compaction requested=false 2024-11-10T04:37:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-10T04:37:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-10T04:37:57,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-10T04:37:57,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 982 msec 2024-11-10T04:37:57,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 989 msec 2024-11-10T04:37:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T04:37:57,482 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:37:57,482 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:37:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-10T04:37:57,485 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:37:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T04:37:57,486 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:37:57,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:37:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:57,521 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:57,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:37:57,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/3a2ea7127e514a26bed963208f103073 is 50, key is test_row_0/A:col10/1731213477520/Put/seqid=0 2024-11-10T04:37:57,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742118_1294 (size=12301) 2024-11-10T04:37:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T04:37:57,624 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/6864d00395ff4a99915e854e9dc1107c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6864d00395ff4a99915e854e9dc1107c 2024-11-10T04:37:57,628 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/A of 305ea5ff5a8e2066f94db3068ceb2842 into 6864d00395ff4a99915e854e9dc1107c(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:57,628 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:57,629 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/A, priority=13, startTime=1731213477203; duration=0sec 2024-11-10T04:37:57,629 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:57,629 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:A 2024-11-10T04:37:57,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213537626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:57,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:57,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:57,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:57,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:57,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,658 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/da0cb9269a2c42d3aa20f51413841db3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/da0cb9269a2c42d3aa20f51413841db3 2024-11-10T04:37:57,663 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/C of 305ea5ff5a8e2066f94db3068ceb2842 into da0cb9269a2c42d3aa20f51413841db3(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:57,663 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:57,663 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/C, priority=13, startTime=1731213477203; duration=0sec 2024-11-10T04:37:57,663 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:57,663 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:C 2024-11-10T04:37:57,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213537741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:57,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:57,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:57,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T04:37:57,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/3a2ea7127e514a26bed963208f103073 2024-11-10T04:37:57,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/6c8753b07c3043f5b05565a50e4a0c23 is 50, key is test_row_0/B:col10/1731213477520/Put/seqid=0 2024-11-10T04:37:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742119_1295 (size=12301) 2024-11-10T04:37:57,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:57,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:57,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213537950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:57,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:58,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:58,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T04:37:58,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213538261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:58,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:58,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/6c8753b07c3043f5b05565a50e4a0c23 2024-11-10T04:37:58,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/8b056fff3244467caa222aea522c860f is 50, key is test_row_0/C:col10/1731213477520/Put/seqid=0 2024-11-10T04:37:58,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742120_1296 (size=12301) 2024-11-10T04:37:58,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T04:37:58,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:58,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:58,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/8b056fff3244467caa222aea522c860f 2024-11-10T04:37:58,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/3a2ea7127e514a26bed963208f103073 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/3a2ea7127e514a26bed963208f103073 2024-11-10T04:37:58,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/3a2ea7127e514a26bed963208f103073, entries=150, sequenceid=375, filesize=12.0 K 2024-11-10T04:37:58,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/6c8753b07c3043f5b05565a50e4a0c23 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6c8753b07c3043f5b05565a50e4a0c23 2024-11-10T04:37:58,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6c8753b07c3043f5b05565a50e4a0c23, entries=150, sequenceid=375, filesize=12.0 K 2024-11-10T04:37:58,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/8b056fff3244467caa222aea522c860f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8b056fff3244467caa222aea522c860f 2024-11-10T04:37:58,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8b056fff3244467caa222aea522c860f, entries=150, sequenceid=375, filesize=12.0 K 2024-11-10T04:37:58,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 305ea5ff5a8e2066f94db3068ceb2842 in 1250ms, sequenceid=375, compaction requested=true 2024-11-10T04:37:58,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:37:58,772 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:37:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:37:58,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:58,772 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:58,773 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:58,773 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:58,773 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/A is initiating minor compaction (all files) 2024-11-10T04:37:58,773 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/B is initiating minor compaction (all files) 2024-11-10T04:37:58,773 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/A in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,773 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/B in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,773 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6864d00395ff4a99915e854e9dc1107c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b0322dea124f46bca4f642bd694b52f5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/3a2ea7127e514a26bed963208f103073] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=36.8 K 2024-11-10T04:37:58,773 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a1428ed3541f46ecb6320cc76507698b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a5b63d0de684e12bc53546afe13a2dc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6c8753b07c3043f5b05565a50e4a0c23] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=36.8 K 2024-11-10T04:37:58,773 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a1428ed3541f46ecb6320cc76507698b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731213476161 2024-11-10T04:37:58,773 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6864d00395ff4a99915e854e9dc1107c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731213476161 2024-11-10T04:37:58,774 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0322dea124f46bca4f642bd694b52f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731213476341 2024-11-10T04:37:58,774 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a5b63d0de684e12bc53546afe13a2dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731213476341 2024-11-10T04:37:58,774 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a2ea7127e514a26bed963208f103073, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731213477504 2024-11-10T04:37:58,774 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c8753b07c3043f5b05565a50e4a0c23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731213477504 2024-11-10T04:37:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:58,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:37:58,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:37:58,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/a5f18b4ef30e47288e5218fdeb620f77 is 50, key is test_row_0/A:col10/1731213478773/Put/seqid=0 2024-11-10T04:37:58,797 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#A#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:58,797 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#B#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:58,798 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/834a0aeeaa9c4be59cc5647286664ef9 is 50, key is test_row_0/A:col10/1731213477520/Put/seqid=0 2024-11-10T04:37:58,798 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/83bae30875bd4fa6b0a43c15b0ed24fc is 50, key is test_row_0/B:col10/1731213477520/Put/seqid=0 2024-11-10T04:37:58,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742123_1299 (size=13221) 2024-11-10T04:37:58,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742121_1297 (size=14741) 2024-11-10T04:37:58,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/a5f18b4ef30e47288e5218fdeb620f77 2024-11-10T04:37:58,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742122_1298 (size=13221) 2024-11-10T04:37:58,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213538810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:58,822 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/83bae30875bd4fa6b0a43c15b0ed24fc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/83bae30875bd4fa6b0a43c15b0ed24fc 2024-11-10T04:37:58,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/662b00cfdf4d42ebacd440a46a72d200 is 50, key is test_row_0/B:col10/1731213478773/Put/seqid=0 2024-11-10T04:37:58,827 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/B of 305ea5ff5a8e2066f94db3068ceb2842 into 83bae30875bd4fa6b0a43c15b0ed24fc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:58,827 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:58,827 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/B, priority=13, startTime=1731213478772; duration=0sec 2024-11-10T04:37:58,827 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:37:58,827 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:B 2024-11-10T04:37:58,827 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:37:58,829 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:37:58,829 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/C is initiating minor compaction (all files) 2024-11-10T04:37:58,829 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/C in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,829 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/da0cb9269a2c42d3aa20f51413841db3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/81b250f67fd64eb79dcd5069c4b01c37, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8b056fff3244467caa222aea522c860f] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=36.8 K 2024-11-10T04:37:58,830 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting da0cb9269a2c42d3aa20f51413841db3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731213476161 2024-11-10T04:37:58,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742124_1300 (size=12301) 2024-11-10T04:37:58,831 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b250f67fd64eb79dcd5069c4b01c37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731213476341 2024-11-10T04:37:58,832 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b056fff3244467caa222aea522c860f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731213477504 2024-11-10T04:37:58,839 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:37:58,839 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/f10f86cf6f544bb9af8a67deed8454b0 is 50, key is test_row_0/C:col10/1731213477520/Put/seqid=0 2024-11-10T04:37:58,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742125_1301 (size=13221) 2024-11-10T04:37:58,848 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/f10f86cf6f544bb9af8a67deed8454b0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f10f86cf6f544bb9af8a67deed8454b0 2024-11-10T04:37:58,854 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/C of 305ea5ff5a8e2066f94db3068ceb2842 into f10f86cf6f544bb9af8a67deed8454b0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:58,854 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:58,854 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/C, priority=13, startTime=1731213478772; duration=0sec 2024-11-10T04:37:58,854 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:58,854 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:C 2024-11-10T04:37:58,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:58,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:58,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213538921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:59,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213539131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:59,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:59,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:59,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,209 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/834a0aeeaa9c4be59cc5647286664ef9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/834a0aeeaa9c4be59cc5647286664ef9 2024-11-10T04:37:59,213 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/A of 305ea5ff5a8e2066f94db3068ceb2842 into 834a0aeeaa9c4be59cc5647286664ef9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:37:59,213 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:59,213 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/A, priority=13, startTime=1731213478771; duration=0sec 2024-11-10T04:37:59,213 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:37:59,213 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:A 2024-11-10T04:37:59,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/662b00cfdf4d42ebacd440a46a72d200 2024-11-10T04:37:59,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/6aef53e9c4134112ba50034edeb46f67 is 50, key is test_row_0/C:col10/1731213478773/Put/seqid=0 2024-11-10T04:37:59,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742126_1302 (size=12301) 2024-11-10T04:37:59,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:59,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:37:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213539441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:37:59,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:59,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:37:59,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:37:59,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:37:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:37:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T04:37:59,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/6aef53e9c4134112ba50034edeb46f67 2024-11-10T04:37:59,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/a5f18b4ef30e47288e5218fdeb620f77 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a5f18b4ef30e47288e5218fdeb620f77 2024-11-10T04:37:59,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a5f18b4ef30e47288e5218fdeb620f77, entries=200, sequenceid=403, filesize=14.4 K 2024-11-10T04:37:59,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/662b00cfdf4d42ebacd440a46a72d200 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/662b00cfdf4d42ebacd440a46a72d200 2024-11-10T04:37:59,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/662b00cfdf4d42ebacd440a46a72d200, entries=150, sequenceid=403, filesize=12.0 K 2024-11-10T04:37:59,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/6aef53e9c4134112ba50034edeb46f67 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6aef53e9c4134112ba50034edeb46f67 2024-11-10T04:37:59,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6aef53e9c4134112ba50034edeb46f67, entries=150, sequenceid=403, filesize=12.0 K 2024-11-10T04:37:59,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 305ea5ff5a8e2066f94db3068ceb2842 in 887ms, sequenceid=403, compaction requested=false 2024-11-10T04:37:59,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:37:59,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T04:37:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:37:59,780 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T04:37:59,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/44934a8939084c90831ce2f86a7b21f6 is 50, key is test_row_0/A:col10/1731213478804/Put/seqid=0 2024-11-10T04:37:59,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742127_1303 (size=9857) 2024-11-10T04:37:59,796 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/44934a8939084c90831ce2f86a7b21f6 2024-11-10T04:37:59,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/56b4a64ceee04403b6353640999f2ce5 is 50, key is test_row_0/B:col10/1731213478804/Put/seqid=0 2024-11-10T04:37:59,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742128_1304 (size=9857) 2024-11-10T04:37:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:37:59,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:38:00,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213540053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:00,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:00,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213540160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:00,209 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/56b4a64ceee04403b6353640999f2ce5 2024-11-10T04:38:00,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/14591ac80b4b411192abcc3185beea5c is 50, key is test_row_0/C:col10/1731213478804/Put/seqid=0 2024-11-10T04:38:00,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742129_1305 (size=9857) 2024-11-10T04:38:00,306 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:38:00,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213540371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:00,621 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/14591ac80b4b411192abcc3185beea5c 2024-11-10T04:38:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/44934a8939084c90831ce2f86a7b21f6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/44934a8939084c90831ce2f86a7b21f6 2024-11-10T04:38:00,630 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/44934a8939084c90831ce2f86a7b21f6, entries=100, sequenceid=416, filesize=9.6 K 2024-11-10T04:38:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/56b4a64ceee04403b6353640999f2ce5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/56b4a64ceee04403b6353640999f2ce5 2024-11-10T04:38:00,634 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/56b4a64ceee04403b6353640999f2ce5, entries=100, sequenceid=416, filesize=9.6 K 2024-11-10T04:38:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/14591ac80b4b411192abcc3185beea5c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/14591ac80b4b411192abcc3185beea5c 2024-11-10T04:38:00,639 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/14591ac80b4b411192abcc3185beea5c, entries=100, sequenceid=416, filesize=9.6 K 2024-11-10T04:38:00,640 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 305ea5ff5a8e2066f94db3068ceb2842 in 860ms, sequenceid=416, compaction requested=true 2024-11-10T04:38:00,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:38:00,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:00,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-10T04:38:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-10T04:38:00,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-10T04:38:00,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1550 sec 2024-11-10T04:38:00,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 3.1610 sec 2024-11-10T04:38:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:38:00,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:00,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T04:38:00,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/f5dfb9b5052445b296d38e97644a2889 is 50, key is test_row_0/A:col10/1731213480046/Put/seqid=0 2024-11-10T04:38:00,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742130_1306 (size=12301) 2024-11-10T04:38:00,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213540708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:00,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213540821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:01,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213541031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:01,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/f5dfb9b5052445b296d38e97644a2889 2024-11-10T04:38:01,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/1b9c00fb6416435399eee8b873e947b5 is 50, key is test_row_0/B:col10/1731213480046/Put/seqid=0 2024-11-10T04:38:01,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742131_1307 (size=12301) 2024-11-10T04:38:01,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213541341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:01,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/1b9c00fb6416435399eee8b873e947b5 2024-11-10T04:38:01,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2cf5998cc417408eb0e00e68690a71c7 is 50, key is test_row_0/C:col10/1731213480046/Put/seqid=0 2024-11-10T04:38:01,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742132_1308 (size=12301) 2024-11-10T04:38:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T04:38:01,661 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:01,662 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-10T04:38:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T04:38:01,663 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:01,664 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:01,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T04:38:01,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-10T04:38:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. as already flushing 2024-11-10T04:38:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:01,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:01,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731213541851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:01,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2cf5998cc417408eb0e00e68690a71c7 2024-11-10T04:38:01,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/f5dfb9b5052445b296d38e97644a2889 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f5dfb9b5052445b296d38e97644a2889 2024-11-10T04:38:01,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f5dfb9b5052445b296d38e97644a2889, entries=150, sequenceid=443, filesize=12.0 K 2024-11-10T04:38:01,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/1b9c00fb6416435399eee8b873e947b5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/1b9c00fb6416435399eee8b873e947b5 2024-11-10T04:38:01,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/1b9c00fb6416435399eee8b873e947b5, entries=150, sequenceid=443, filesize=12.0 K 2024-11-10T04:38:01,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2cf5998cc417408eb0e00e68690a71c7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2cf5998cc417408eb0e00e68690a71c7 2024-11-10T04:38:01,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2cf5998cc417408eb0e00e68690a71c7, entries=150, sequenceid=443, filesize=12.0 K 2024-11-10T04:38:01,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 305ea5ff5a8e2066f94db3068ceb2842 in 1253ms, sequenceid=443, compaction requested=true 2024-11-10T04:38:01,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:38:01,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:01,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:01,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:01,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:01,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ea5ff5a8e2066f94db3068ceb2842:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:01,936 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:01,936 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:01,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:01,938 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50120 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:01,938 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:01,938 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/B is initiating minor compaction (all files) 2024-11-10T04:38:01,938 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/A is initiating minor compaction (all files) 2024-11-10T04:38:01,938 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/B in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:01,938 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/A in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:01,938 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/83bae30875bd4fa6b0a43c15b0ed24fc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/662b00cfdf4d42ebacd440a46a72d200, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/56b4a64ceee04403b6353640999f2ce5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/1b9c00fb6416435399eee8b873e947b5] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=46.6 K 2024-11-10T04:38:01,938 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/834a0aeeaa9c4be59cc5647286664ef9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a5f18b4ef30e47288e5218fdeb620f77, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/44934a8939084c90831ce2f86a7b21f6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f5dfb9b5052445b296d38e97644a2889] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=48.9 K 2024-11-10T04:38:01,938 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 83bae30875bd4fa6b0a43c15b0ed24fc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731213477504 2024-11-10T04:38:01,939 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834a0aeeaa9c4be59cc5647286664ef9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731213477504 2024-11-10T04:38:01,939 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5f18b4ef30e47288e5218fdeb620f77, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731213477612 2024-11-10T04:38:01,939 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 662b00cfdf4d42ebacd440a46a72d200, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731213477618 2024-11-10T04:38:01,939 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 56b4a64ceee04403b6353640999f2ce5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731213478804 2024-11-10T04:38:01,940 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44934a8939084c90831ce2f86a7b21f6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731213478804 2024-11-10T04:38:01,940 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9c00fb6416435399eee8b873e947b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731213480046 2024-11-10T04:38:01,940 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5dfb9b5052445b296d38e97644a2889, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731213480046 2024-11-10T04:38:01,949 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:01,950 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/5a736b076d334fbb9fcff20371ae00ed is 50, key is test_row_0/B:col10/1731213480046/Put/seqid=0 2024-11-10T04:38:01,950 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:01,950 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/5dff614d0c73481bbecaeddc679b1ec8 is 50, key is test_row_0/A:col10/1731213480046/Put/seqid=0 2024-11-10T04:38:01,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742133_1309 (size=13357) 2024-11-10T04:38:01,970 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/5dff614d0c73481bbecaeddc679b1ec8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/5dff614d0c73481bbecaeddc679b1ec8 2024-11-10T04:38:01,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-10T04:38:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:01,970 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-10T04:38:01,975 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/A of 305ea5ff5a8e2066f94db3068ceb2842 into 5dff614d0c73481bbecaeddc679b1ec8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:01,975 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:38:01,975 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/A, priority=12, startTime=1731213481936; duration=0sec 2024-11-10T04:38:01,975 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:01,975 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:A 2024-11-10T04:38:01,975 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:01,976 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:01,976 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 305ea5ff5a8e2066f94db3068ceb2842/C is initiating minor compaction (all files) 2024-11-10T04:38:01,976 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ea5ff5a8e2066f94db3068ceb2842/C in TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:01,976 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f10f86cf6f544bb9af8a67deed8454b0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6aef53e9c4134112ba50034edeb46f67, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/14591ac80b4b411192abcc3185beea5c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2cf5998cc417408eb0e00e68690a71c7] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp, totalSize=46.6 K 2024-11-10T04:38:01,977 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f10f86cf6f544bb9af8a67deed8454b0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731213477504 2024-11-10T04:38:01,977 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aef53e9c4134112ba50034edeb46f67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731213477618 2024-11-10T04:38:01,977 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14591ac80b4b411192abcc3185beea5c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731213478804 2024-11-10T04:38:01,978 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cf5998cc417408eb0e00e68690a71c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731213480046 2024-11-10T04:38:01,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742134_1310 (size=13357) 2024-11-10T04:38:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/87b53bca0cb74f6a951764cf655474be is 50, key is test_row_0/A:col10/1731213480707/Put/seqid=0 2024-11-10T04:38:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T04:38:01,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742135_1311 (size=9857) 2024-11-10T04:38:01,993 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ea5ff5a8e2066f94db3068ceb2842#C#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:01,994 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/fd280ccd538a4734aa7797ece55f4843 is 50, key is test_row_0/C:col10/1731213480046/Put/seqid=0 2024-11-10T04:38:02,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742136_1312 (size=13357) 2024-11-10T04:38:02,011 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/fd280ccd538a4734aa7797ece55f4843 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/fd280ccd538a4734aa7797ece55f4843 2024-11-10T04:38:02,017 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/C of 305ea5ff5a8e2066f94db3068ceb2842 into fd280ccd538a4734aa7797ece55f4843(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:02,017 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:38:02,017 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/C, priority=12, startTime=1731213481936; duration=0sec 2024-11-10T04:38:02,017 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:02,017 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:C 2024-11-10T04:38:02,043 INFO [Thread-1034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1034. 2024-11-10T04:38:02,043 DEBUG [Thread-1034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:02,043 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:02,044 INFO [Thread-1036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1036. 2024-11-10T04:38:02,044 DEBUG [Thread-1036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:02,044 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,044 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,044 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:02,045 INFO [Thread-1040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1040. 2024-11-10T04:38:02,045 DEBUG [Thread-1040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:02,045 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,045 INFO [Thread-1032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1032. 2024-11-10T04:38:02,045 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,045 DEBUG [Thread-1032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:02,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:02,045 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:02,045 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,046 INFO [Thread-1038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1038. 2024-11-10T04:38:02,046 DEBUG [Thread-1038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:02,046 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,047 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T04:38:02,383 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/5a736b076d334fbb9fcff20371ae00ed as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/5a736b076d334fbb9fcff20371ae00ed 2024-11-10T04:38:02,387 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ea5ff5a8e2066f94db3068ceb2842/B of 305ea5ff5a8e2066f94db3068ceb2842 into 5a736b076d334fbb9fcff20371ae00ed(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:02,387 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:38:02,387 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., storeName=305ea5ff5a8e2066f94db3068ceb2842/B, priority=12, startTime=1731213481936; duration=0sec 2024-11-10T04:38:02,387 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:02,387 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ea5ff5a8e2066f94db3068ceb2842:B 2024-11-10T04:38:02,390 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/87b53bca0cb74f6a951764cf655474be 2024-11-10T04:38:02,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/bacd94427332490d9dd5a6d02c2f2460 is 50, key is test_row_0/B:col10/1731213480707/Put/seqid=0 2024-11-10T04:38:02,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742137_1313 (size=9857) 2024-11-10T04:38:02,799 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/bacd94427332490d9dd5a6d02c2f2460 2024-11-10T04:38:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T04:38:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2b4b0071723748e4bd9c29c192b8c814 is 50, key is test_row_0/C:col10/1731213480707/Put/seqid=0 2024-11-10T04:38:02,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742138_1314 (size=9857) 2024-11-10T04:38:02,862 INFO [Thread-1025 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1025. 2024-11-10T04:38:02,862 DEBUG [Thread-1025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:02,862 DEBUG [Thread-1025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,863 DEBUG [Thread-1025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:02,863 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:03,209 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2b4b0071723748e4bd9c29c192b8c814 2024-11-10T04:38:03,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/87b53bca0cb74f6a951764cf655474be as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/87b53bca0cb74f6a951764cf655474be 2024-11-10T04:38:03,217 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/87b53bca0cb74f6a951764cf655474be, entries=100, sequenceid=452, filesize=9.6 K 2024-11-10T04:38:03,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/bacd94427332490d9dd5a6d02c2f2460 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/bacd94427332490d9dd5a6d02c2f2460 2024-11-10T04:38:03,221 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/bacd94427332490d9dd5a6d02c2f2460, entries=100, sequenceid=452, filesize=9.6 K 2024-11-10T04:38:03,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/2b4b0071723748e4bd9c29c192b8c814 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2b4b0071723748e4bd9c29c192b8c814 2024-11-10T04:38:03,224 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2b4b0071723748e4bd9c29c192b8c814, entries=100, sequenceid=452, filesize=9.6 K 2024-11-10T04:38:03,225 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=6.71 KB/6870 for 305ea5ff5a8e2066f94db3068ceb2842 in 1255ms, sequenceid=452, compaction requested=false 2024-11-10T04:38:03,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 305ea5ff5a8e2066f94db3068ceb2842: 2024-11-10T04:38:03,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:03,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-10T04:38:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-10T04:38:03,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-10T04:38:03,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5620 sec 2024-11-10T04:38:03,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.5660 sec 2024-11-10T04:38:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T04:38:03,812 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:05,212 INFO [Thread-1021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1021. 2024-11-10T04:38:05,212 DEBUG [Thread-1021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:05,212 DEBUG [Thread-1021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:05,213 DEBUG [Thread-1021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:05,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:06,353 INFO [Thread-1023 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1023. 2024-11-10T04:38:06,353 DEBUG [Thread-1023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:06,353 DEBUG [Thread-1023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:06,353 DEBUG [Thread-1023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:06,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:06,363 INFO [Thread-1029 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1029. 2024-11-10T04:38:06,363 DEBUG [Thread-1029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:06,363 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:06,363 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:06,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:06,392 INFO [Thread-1027 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1027. 2024-11-10T04:38:06,392 DEBUG [Thread-1027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:06,392 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:06,392 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:06,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 243 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3774 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3807 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4009 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3760 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3809 2024-11-10T04:38:06,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T04:38:06,392 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:38:06,392 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:38:06,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:06,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:06,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:06,393 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T04:38:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T04:38:06,396 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213486396"}]},"ts":"1731213486396"} 2024-11-10T04:38:06,397 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T04:38:06,397 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T04:38:06,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T04:38:06,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ea5ff5a8e2066f94db3068ceb2842, UNASSIGN}] 2024-11-10T04:38:06,399 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ea5ff5a8e2066f94db3068ceb2842, UNASSIGN 2024-11-10T04:38:06,400 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=305ea5ff5a8e2066f94db3068ceb2842, regionState=CLOSING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:06,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ea5ff5a8e2066f94db3068ceb2842, UNASSIGN because future has completed 2024-11-10T04:38:06,401 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T04:38:06,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:38:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T04:38:06,553 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:38:06,553 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T04:38:06,553 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 305ea5ff5a8e2066f94db3068ceb2842, disabling compactions & flushes 2024-11-10T04:38:06,553 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:06,553 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:06,553 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. after waiting 0 ms 2024-11-10T04:38:06,553 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:06,553 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(2902): Flushing 305ea5ff5a8e2066f94db3068ceb2842 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T04:38:06,556 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/89c47142a5a840e3bff502e8f59abddd is 50, key is test_row_1/A:col10/1731213486351/Put/seqid=0 2024-11-10T04:38:06,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742139_1315 (size=9857) 2024-11-10T04:38:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T04:38:06,961 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/89c47142a5a840e3bff502e8f59abddd 2024-11-10T04:38:06,968 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/442cfbe2255346dab05a14f285cb6649 is 50, key is test_row_1/B:col10/1731213486351/Put/seqid=0 2024-11-10T04:38:06,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742140_1316 (size=9857) 2024-11-10T04:38:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T04:38:07,372 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/442cfbe2255346dab05a14f285cb6649 2024-11-10T04:38:07,378 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/b299b3af04d24987b9785e99f06d70b8 is 50, key is test_row_1/C:col10/1731213486351/Put/seqid=0 2024-11-10T04:38:07,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742141_1317 (size=9857) 2024-11-10T04:38:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T04:38:07,782 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/b299b3af04d24987b9785e99f06d70b8 2024-11-10T04:38:07,786 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/A/89c47142a5a840e3bff502e8f59abddd as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/89c47142a5a840e3bff502e8f59abddd 2024-11-10T04:38:07,790 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/89c47142a5a840e3bff502e8f59abddd, entries=100, sequenceid=463, filesize=9.6 K 2024-11-10T04:38:07,791 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/B/442cfbe2255346dab05a14f285cb6649 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/442cfbe2255346dab05a14f285cb6649 2024-11-10T04:38:07,794 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/442cfbe2255346dab05a14f285cb6649, entries=100, sequenceid=463, filesize=9.6 K 2024-11-10T04:38:07,795 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/.tmp/C/b299b3af04d24987b9785e99f06d70b8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b299b3af04d24987b9785e99f06d70b8 2024-11-10T04:38:07,798 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b299b3af04d24987b9785e99f06d70b8, entries=100, sequenceid=463, filesize=9.6 K 2024-11-10T04:38:07,798 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 305ea5ff5a8e2066f94db3068ceb2842 in 1245ms, sequenceid=463, compaction requested=true 2024-11-10T04:38:07,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/8bb1c97e732d46d38cdfddf07414ae63, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f91922f4fe4f448ebfec740e0cd4895a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a03d0b38999548c8b88f05929822ec76, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/489a278991f246ddba3865bab93d1d74, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/19bdedf0e291431a82bf00fa35f71db0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6ed60759f7de45e6afc4feb34218ee01, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/87c09918febe428cbe01e1e5b1aed8b5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a79c8a40cd7c49a0975f0a005c923611, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7f26a420f1ec4bc987617032e1152fc9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/86a9f758380a437680f518388c8fa26e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/756f97604fb04acc83485943ac3b310a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/237e7ef235a94e0d99ef75f10f5abec5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/84178bf4232943eaafbf58f8d153ce43, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7ad927e2c9b540e1b2101e51efba67b5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/ae28bee4094c487e9f7308bc54aec2aa, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/88d671dbb8c442028363eca8c7f9e702, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/e426929e1a504df28723e72127f99c44, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/54f0288a00894d9b85cebb0138057de6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/27da507eb78d4651a0e3ec2b2ab9e104, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b829af1a6fb24b82a806df560b0c91f5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/110854ef0bd44e2aa511f9400d22fc2f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/c56d6e7d47ec4c9aad9fbf5e4720f164, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/1355acc051f24eec9f439b5b1f96e049, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/05b938b0a8d14c619c6f5320c92fe3d9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6864d00395ff4a99915e854e9dc1107c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b0322dea124f46bca4f642bd694b52f5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/834a0aeeaa9c4be59cc5647286664ef9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/3a2ea7127e514a26bed963208f103073, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a5f18b4ef30e47288e5218fdeb620f77, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/44934a8939084c90831ce2f86a7b21f6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f5dfb9b5052445b296d38e97644a2889] to archive 2024-11-10T04:38:07,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:38:07,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/8bb1c97e732d46d38cdfddf07414ae63 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/8bb1c97e732d46d38cdfddf07414ae63 2024-11-10T04:38:07,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f91922f4fe4f448ebfec740e0cd4895a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f91922f4fe4f448ebfec740e0cd4895a 2024-11-10T04:38:07,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a03d0b38999548c8b88f05929822ec76 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a03d0b38999548c8b88f05929822ec76 2024-11-10T04:38:07,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/489a278991f246ddba3865bab93d1d74 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/489a278991f246ddba3865bab93d1d74 2024-11-10T04:38:07,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/19bdedf0e291431a82bf00fa35f71db0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/19bdedf0e291431a82bf00fa35f71db0 2024-11-10T04:38:07,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6ed60759f7de45e6afc4feb34218ee01 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6ed60759f7de45e6afc4feb34218ee01 2024-11-10T04:38:07,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/87c09918febe428cbe01e1e5b1aed8b5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/87c09918febe428cbe01e1e5b1aed8b5 2024-11-10T04:38:07,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a79c8a40cd7c49a0975f0a005c923611 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a79c8a40cd7c49a0975f0a005c923611 2024-11-10T04:38:07,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7f26a420f1ec4bc987617032e1152fc9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7f26a420f1ec4bc987617032e1152fc9 2024-11-10T04:38:07,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/86a9f758380a437680f518388c8fa26e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/86a9f758380a437680f518388c8fa26e 2024-11-10T04:38:07,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/756f97604fb04acc83485943ac3b310a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/756f97604fb04acc83485943ac3b310a 2024-11-10T04:38:07,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/237e7ef235a94e0d99ef75f10f5abec5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/237e7ef235a94e0d99ef75f10f5abec5 2024-11-10T04:38:07,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/84178bf4232943eaafbf58f8d153ce43 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/84178bf4232943eaafbf58f8d153ce43 2024-11-10T04:38:07,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7ad927e2c9b540e1b2101e51efba67b5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/7ad927e2c9b540e1b2101e51efba67b5 2024-11-10T04:38:07,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/ae28bee4094c487e9f7308bc54aec2aa to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/ae28bee4094c487e9f7308bc54aec2aa 2024-11-10T04:38:07,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/88d671dbb8c442028363eca8c7f9e702 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/88d671dbb8c442028363eca8c7f9e702 2024-11-10T04:38:07,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/e426929e1a504df28723e72127f99c44 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/e426929e1a504df28723e72127f99c44 2024-11-10T04:38:07,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/54f0288a00894d9b85cebb0138057de6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/54f0288a00894d9b85cebb0138057de6 2024-11-10T04:38:07,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/27da507eb78d4651a0e3ec2b2ab9e104 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/27da507eb78d4651a0e3ec2b2ab9e104 2024-11-10T04:38:07,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b829af1a6fb24b82a806df560b0c91f5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b829af1a6fb24b82a806df560b0c91f5 2024-11-10T04:38:07,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/110854ef0bd44e2aa511f9400d22fc2f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/110854ef0bd44e2aa511f9400d22fc2f 2024-11-10T04:38:07,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/c56d6e7d47ec4c9aad9fbf5e4720f164 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/c56d6e7d47ec4c9aad9fbf5e4720f164 2024-11-10T04:38:07,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/1355acc051f24eec9f439b5b1f96e049 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/1355acc051f24eec9f439b5b1f96e049 2024-11-10T04:38:07,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/05b938b0a8d14c619c6f5320c92fe3d9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/05b938b0a8d14c619c6f5320c92fe3d9 2024-11-10T04:38:07,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6864d00395ff4a99915e854e9dc1107c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/6864d00395ff4a99915e854e9dc1107c 2024-11-10T04:38:07,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b0322dea124f46bca4f642bd694b52f5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/b0322dea124f46bca4f642bd694b52f5 2024-11-10T04:38:07,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/834a0aeeaa9c4be59cc5647286664ef9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/834a0aeeaa9c4be59cc5647286664ef9 2024-11-10T04:38:07,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/3a2ea7127e514a26bed963208f103073 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/3a2ea7127e514a26bed963208f103073 2024-11-10T04:38:07,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a5f18b4ef30e47288e5218fdeb620f77 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/a5f18b4ef30e47288e5218fdeb620f77 2024-11-10T04:38:07,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/44934a8939084c90831ce2f86a7b21f6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/44934a8939084c90831ce2f86a7b21f6 2024-11-10T04:38:07,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f5dfb9b5052445b296d38e97644a2889 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/f5dfb9b5052445b296d38e97644a2889 2024-11-10T04:38:07,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d7908dc2e7d44544bc7ab10ea4cba02b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/5b0cad92709845a4b1fe1a852087f69f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/31c135312d8d461f9a85080c497b501e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4f3026b4dc574586b0e87aed4bc86070, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/11d1d0d24cba4befbf77bf9e3741df93, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8088d2e1800d4a21bb06c92fcb0d9dba, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a86f16f465674f6eadb5374a26b9fd8a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/57fbee14ea9f49c59674d281fcff1b09, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d83ca5fc630b4fdc9044f9e36fe8f768, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6b84d5fe93e04d8fa539d6b8d42f44a2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e9780db6b48d42548d5f4c9975018c36, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fce5720ff13b414db0cb3eb91a9cc2fc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/ca70fd5b6f9747eab523cf862ca425df, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fec2b2a30ea84975b465216235f9fe85, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/f74fa30ffcf34f46bd0a441f6d582c92, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/c3d4a2ae35f34dc98f3f11b25b04fc07, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a35ee88388446d1985068da085be651, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/7ed362ed8d0641a8bcc39ba6c66c2bdb, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e6e1178780c6473883cef4c9745427e5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/cfb63325184349a79df52ac49b643d28, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a7c8c93e72cd410a9a180a8e86779ca7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/af98b641c93744fa9bb56883736f2eb7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/711f5724685d4ae79f3f76fd0dd30b51, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a1428ed3541f46ecb6320cc76507698b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8d2a71284faf4c8baae072194743dce2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a5b63d0de684e12bc53546afe13a2dc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/83bae30875bd4fa6b0a43c15b0ed24fc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6c8753b07c3043f5b05565a50e4a0c23, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/662b00cfdf4d42ebacd440a46a72d200, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/56b4a64ceee04403b6353640999f2ce5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/1b9c00fb6416435399eee8b873e947b5] to archive 2024-11-10T04:38:07,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:38:07,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d7908dc2e7d44544bc7ab10ea4cba02b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d7908dc2e7d44544bc7ab10ea4cba02b 2024-11-10T04:38:07,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/5b0cad92709845a4b1fe1a852087f69f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/5b0cad92709845a4b1fe1a852087f69f 2024-11-10T04:38:07,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/31c135312d8d461f9a85080c497b501e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/31c135312d8d461f9a85080c497b501e 2024-11-10T04:38:07,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4f3026b4dc574586b0e87aed4bc86070 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4f3026b4dc574586b0e87aed4bc86070 2024-11-10T04:38:07,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/11d1d0d24cba4befbf77bf9e3741df93 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/11d1d0d24cba4befbf77bf9e3741df93 2024-11-10T04:38:07,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8088d2e1800d4a21bb06c92fcb0d9dba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8088d2e1800d4a21bb06c92fcb0d9dba 2024-11-10T04:38:07,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a86f16f465674f6eadb5374a26b9fd8a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a86f16f465674f6eadb5374a26b9fd8a 2024-11-10T04:38:07,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/57fbee14ea9f49c59674d281fcff1b09 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/57fbee14ea9f49c59674d281fcff1b09 2024-11-10T04:38:07,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d83ca5fc630b4fdc9044f9e36fe8f768 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/d83ca5fc630b4fdc9044f9e36fe8f768 2024-11-10T04:38:07,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6b84d5fe93e04d8fa539d6b8d42f44a2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6b84d5fe93e04d8fa539d6b8d42f44a2 2024-11-10T04:38:07,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e9780db6b48d42548d5f4c9975018c36 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e9780db6b48d42548d5f4c9975018c36 2024-11-10T04:38:07,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fce5720ff13b414db0cb3eb91a9cc2fc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fce5720ff13b414db0cb3eb91a9cc2fc 2024-11-10T04:38:07,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/ca70fd5b6f9747eab523cf862ca425df to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/ca70fd5b6f9747eab523cf862ca425df 2024-11-10T04:38:07,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fec2b2a30ea84975b465216235f9fe85 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/fec2b2a30ea84975b465216235f9fe85 2024-11-10T04:38:07,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/f74fa30ffcf34f46bd0a441f6d582c92 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/f74fa30ffcf34f46bd0a441f6d582c92 2024-11-10T04:38:07,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/c3d4a2ae35f34dc98f3f11b25b04fc07 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/c3d4a2ae35f34dc98f3f11b25b04fc07 2024-11-10T04:38:07,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a35ee88388446d1985068da085be651 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a35ee88388446d1985068da085be651 2024-11-10T04:38:07,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/7ed362ed8d0641a8bcc39ba6c66c2bdb to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/7ed362ed8d0641a8bcc39ba6c66c2bdb 2024-11-10T04:38:07,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e6e1178780c6473883cef4c9745427e5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/e6e1178780c6473883cef4c9745427e5 2024-11-10T04:38:07,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/cfb63325184349a79df52ac49b643d28 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/cfb63325184349a79df52ac49b643d28 2024-11-10T04:38:07,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a7c8c93e72cd410a9a180a8e86779ca7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a7c8c93e72cd410a9a180a8e86779ca7 2024-11-10T04:38:07,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/af98b641c93744fa9bb56883736f2eb7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/af98b641c93744fa9bb56883736f2eb7 2024-11-10T04:38:07,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/711f5724685d4ae79f3f76fd0dd30b51 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/711f5724685d4ae79f3f76fd0dd30b51 2024-11-10T04:38:07,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a1428ed3541f46ecb6320cc76507698b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/a1428ed3541f46ecb6320cc76507698b 2024-11-10T04:38:07,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8d2a71284faf4c8baae072194743dce2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/8d2a71284faf4c8baae072194743dce2 2024-11-10T04:38:07,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a5b63d0de684e12bc53546afe13a2dc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/4a5b63d0de684e12bc53546afe13a2dc 2024-11-10T04:38:07,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/83bae30875bd4fa6b0a43c15b0ed24fc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/83bae30875bd4fa6b0a43c15b0ed24fc 2024-11-10T04:38:07,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6c8753b07c3043f5b05565a50e4a0c23 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/6c8753b07c3043f5b05565a50e4a0c23 2024-11-10T04:38:07,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/662b00cfdf4d42ebacd440a46a72d200 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/662b00cfdf4d42ebacd440a46a72d200 2024-11-10T04:38:07,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/56b4a64ceee04403b6353640999f2ce5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/56b4a64ceee04403b6353640999f2ce5 2024-11-10T04:38:07,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/1b9c00fb6416435399eee8b873e947b5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/1b9c00fb6416435399eee8b873e947b5 2024-11-10T04:38:07,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/a98f040ba7b442fa96642c0d312e8e77, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0774fecf2dc8437d9c522c08169de634, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b14e83c03bd845f0b66447258c73877b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8cde5156cff749a6b9819d240683f236, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/1f175955c0084987b730522ec1bf5763, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5375759b318748eaaff9975d0d929b2a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2d1250834dcc439aba5424ca4085ee46, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/d3b335108ea34081883e381719b6b141, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/38274e240b284825b3ad8016bf801c7e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/302c3bc1307847afb2c687f62560f299, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2a31b9b636434dc68886a076a524c942, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e0dd3aa2452842a6907f8b9c0453c4c9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/3cb1b8acb9b644558411b37d9d0c5575, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0deb31e7cad247dba8ffef2229029ff4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b4f51dafe6184bddad90ce4d4a6cac01, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8ae18659cb7c44deafdbe8ebb6e20d08, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/910adde0118143328852673c2ce5c829, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e4a7d5e781154a9bb4e2a9e608d6e52a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6750b9da36dc4b09ac69d72fa75ea0db, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/567ca2302dca4a998d4d75511fc3144f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5a1673914aaa4df39b5ea4d6cbaa86e2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e140a68031814933902398c0fa0f8f20, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/da0cb9269a2c42d3aa20f51413841db3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/88005617f7b641148fdf6537395a389b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/81b250f67fd64eb79dcd5069c4b01c37, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f10f86cf6f544bb9af8a67deed8454b0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8b056fff3244467caa222aea522c860f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6aef53e9c4134112ba50034edeb46f67, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/14591ac80b4b411192abcc3185beea5c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2cf5998cc417408eb0e00e68690a71c7] to archive 2024-11-10T04:38:07,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:38:07,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/a98f040ba7b442fa96642c0d312e8e77 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/a98f040ba7b442fa96642c0d312e8e77 2024-11-10T04:38:07,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f3c9ddf9ef6b4b3c8fd4b520bd6bdfe4 2024-11-10T04:38:07,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0774fecf2dc8437d9c522c08169de634 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0774fecf2dc8437d9c522c08169de634 2024-11-10T04:38:07,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b14e83c03bd845f0b66447258c73877b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b14e83c03bd845f0b66447258c73877b 2024-11-10T04:38:07,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8cde5156cff749a6b9819d240683f236 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8cde5156cff749a6b9819d240683f236 2024-11-10T04:38:07,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/1f175955c0084987b730522ec1bf5763 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/1f175955c0084987b730522ec1bf5763 2024-11-10T04:38:07,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5375759b318748eaaff9975d0d929b2a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5375759b318748eaaff9975d0d929b2a 2024-11-10T04:38:07,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2d1250834dcc439aba5424ca4085ee46 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2d1250834dcc439aba5424ca4085ee46 2024-11-10T04:38:07,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/d3b335108ea34081883e381719b6b141 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/d3b335108ea34081883e381719b6b141 2024-11-10T04:38:07,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/38274e240b284825b3ad8016bf801c7e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/38274e240b284825b3ad8016bf801c7e 2024-11-10T04:38:07,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/302c3bc1307847afb2c687f62560f299 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/302c3bc1307847afb2c687f62560f299 2024-11-10T04:38:07,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2a31b9b636434dc68886a076a524c942 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2a31b9b636434dc68886a076a524c942 2024-11-10T04:38:07,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e0dd3aa2452842a6907f8b9c0453c4c9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e0dd3aa2452842a6907f8b9c0453c4c9 2024-11-10T04:38:07,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/3cb1b8acb9b644558411b37d9d0c5575 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/3cb1b8acb9b644558411b37d9d0c5575 2024-11-10T04:38:07,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0deb31e7cad247dba8ffef2229029ff4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/0deb31e7cad247dba8ffef2229029ff4 2024-11-10T04:38:07,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b4f51dafe6184bddad90ce4d4a6cac01 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b4f51dafe6184bddad90ce4d4a6cac01 2024-11-10T04:38:07,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8ae18659cb7c44deafdbe8ebb6e20d08 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8ae18659cb7c44deafdbe8ebb6e20d08 2024-11-10T04:38:07,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/910adde0118143328852673c2ce5c829 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/910adde0118143328852673c2ce5c829 2024-11-10T04:38:07,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e4a7d5e781154a9bb4e2a9e608d6e52a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e4a7d5e781154a9bb4e2a9e608d6e52a 2024-11-10T04:38:07,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6750b9da36dc4b09ac69d72fa75ea0db to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6750b9da36dc4b09ac69d72fa75ea0db 2024-11-10T04:38:07,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/567ca2302dca4a998d4d75511fc3144f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/567ca2302dca4a998d4d75511fc3144f 2024-11-10T04:38:07,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5a1673914aaa4df39b5ea4d6cbaa86e2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/5a1673914aaa4df39b5ea4d6cbaa86e2 2024-11-10T04:38:07,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e140a68031814933902398c0fa0f8f20 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/e140a68031814933902398c0fa0f8f20 2024-11-10T04:38:07,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/da0cb9269a2c42d3aa20f51413841db3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/da0cb9269a2c42d3aa20f51413841db3 2024-11-10T04:38:07,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/88005617f7b641148fdf6537395a389b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/88005617f7b641148fdf6537395a389b 2024-11-10T04:38:07,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/81b250f67fd64eb79dcd5069c4b01c37 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/81b250f67fd64eb79dcd5069c4b01c37 2024-11-10T04:38:07,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f10f86cf6f544bb9af8a67deed8454b0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/f10f86cf6f544bb9af8a67deed8454b0 2024-11-10T04:38:07,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8b056fff3244467caa222aea522c860f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/8b056fff3244467caa222aea522c860f 2024-11-10T04:38:07,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6aef53e9c4134112ba50034edeb46f67 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/6aef53e9c4134112ba50034edeb46f67 2024-11-10T04:38:07,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/14591ac80b4b411192abcc3185beea5c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/14591ac80b4b411192abcc3185beea5c 2024-11-10T04:38:07,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2cf5998cc417408eb0e00e68690a71c7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2cf5998cc417408eb0e00e68690a71c7 2024-11-10T04:38:07,913 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/recovered.edits/466.seqid, newMaxSeqId=466, maxSeqId=1 2024-11-10T04:38:07,914 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. 2024-11-10T04:38:07,914 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 305ea5ff5a8e2066f94db3068ceb2842: Waiting for close lock at 1731213486553Running coprocessor pre-close hooks at 1731213486553Disabling compacts and flushes for region at 1731213486553Disabling writes for close at 1731213486553Obtaining lock to block concurrent updates at 1731213486553Preparing flush snapshotting stores in 305ea5ff5a8e2066f94db3068ceb2842 at 1731213486553Finished memstore snapshotting TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731213486553Flushing stores of TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842. at 1731213486554 (+1 ms)Flushing 305ea5ff5a8e2066f94db3068ceb2842/A: creating writer at 1731213486554Flushing 305ea5ff5a8e2066f94db3068ceb2842/A: appending metadata at 1731213486556 (+2 ms)Flushing 305ea5ff5a8e2066f94db3068ceb2842/A: closing flushed file at 1731213486556Flushing 305ea5ff5a8e2066f94db3068ceb2842/B: creating writer at 1731213486965 (+409 ms)Flushing 305ea5ff5a8e2066f94db3068ceb2842/B: appending metadata at 1731213486967 (+2 ms)Flushing 305ea5ff5a8e2066f94db3068ceb2842/B: closing flushed file at 1731213486968 (+1 ms)Flushing 305ea5ff5a8e2066f94db3068ceb2842/C: creating writer at 1731213487375 (+407 ms)Flushing 305ea5ff5a8e2066f94db3068ceb2842/C: appending metadata at 1731213487378 (+3 ms)Flushing 305ea5ff5a8e2066f94db3068ceb2842/C: closing flushed file at 1731213487378Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25e7f14b: reopening flushed file at 1731213487786 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18f4f81: reopening flushed file at 1731213487790 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f67b35b: reopening flushed file at 1731213487794 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 305ea5ff5a8e2066f94db3068ceb2842 in 1245ms, sequenceid=463, compaction requested=true at 1731213487798 (+4 ms)Writing region close event to WAL at 1731213487909 (+111 ms)Running coprocessor post-close hooks at 1731213487913 (+4 ms)Closed at 1731213487914 (+1 ms) 2024-11-10T04:38:07,916 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:38:07,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=305ea5ff5a8e2066f94db3068ceb2842, regionState=CLOSED 2024-11-10T04:38:07,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:38:07,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-10T04:38:07,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 305ea5ff5a8e2066f94db3068ceb2842, server=d1d9e3766cb1,45693,1731213391842 in 1.5170 sec 2024-11-10T04:38:07,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-10T04:38:07,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ea5ff5a8e2066f94db3068ceb2842, UNASSIGN in 1.5220 sec 2024-11-10T04:38:07,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-10T04:38:07,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5250 sec 2024-11-10T04:38:07,925 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213487925"}]},"ts":"1731213487925"} 2024-11-10T04:38:07,927 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T04:38:07,927 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T04:38:07,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5340 sec 2024-11-10T04:38:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T04:38:08,542 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:08,542 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T04:38:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:08,544 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-10T04:38:08,545 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=94, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:08,546 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:38:08,548 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/recovered.edits] 2024-11-10T04:38:08,550 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/5dff614d0c73481bbecaeddc679b1ec8 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/5dff614d0c73481bbecaeddc679b1ec8 2024-11-10T04:38:08,551 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/87b53bca0cb74f6a951764cf655474be to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/87b53bca0cb74f6a951764cf655474be 2024-11-10T04:38:08,552 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/89c47142a5a840e3bff502e8f59abddd to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/A/89c47142a5a840e3bff502e8f59abddd 2024-11-10T04:38:08,554 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/442cfbe2255346dab05a14f285cb6649 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/442cfbe2255346dab05a14f285cb6649 2024-11-10T04:38:08,555 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/5a736b076d334fbb9fcff20371ae00ed to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/5a736b076d334fbb9fcff20371ae00ed 2024-11-10T04:38:08,555 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/bacd94427332490d9dd5a6d02c2f2460 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/B/bacd94427332490d9dd5a6d02c2f2460 2024-11-10T04:38:08,557 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2b4b0071723748e4bd9c29c192b8c814 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/2b4b0071723748e4bd9c29c192b8c814 2024-11-10T04:38:08,558 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b299b3af04d24987b9785e99f06d70b8 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/b299b3af04d24987b9785e99f06d70b8 2024-11-10T04:38:08,559 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/fd280ccd538a4734aa7797ece55f4843 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/C/fd280ccd538a4734aa7797ece55f4843 2024-11-10T04:38:08,561 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/recovered.edits/466.seqid to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842/recovered.edits/466.seqid 2024-11-10T04:38:08,562 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/305ea5ff5a8e2066f94db3068ceb2842 2024-11-10T04:38:08,562 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T04:38:08,564 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=94, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:08,566 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T04:38:08,567 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T04:38:08,568 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=94, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:08,568 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T04:38:08,569 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731213488568"}]},"ts":"9223372036854775807"} 2024-11-10T04:38:08,570 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T04:38:08,570 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 305ea5ff5a8e2066f94db3068ceb2842, NAME => 'TestAcidGuarantees,,1731213459851.305ea5ff5a8e2066f94db3068ceb2842.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T04:38:08,570 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T04:38:08,570 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731213488570"}]},"ts":"9223372036854775807"} 2024-11-10T04:38:08,572 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T04:38:08,572 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=94, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:08,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-10T04:38:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-10T04:38:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T04:38:08,652 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:08,661 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 242), OpenFileDescriptor=455 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=258 (was 214) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6402 (was 6467) 2024-11-10T04:38:08,669 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=6401 2024-11-10T04:38:08,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T04:38:08,670 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:38:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:08,672 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:38:08,673 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:08,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 95 2024-11-10T04:38:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T04:38:08,673 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:38:08,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742142_1318 (size=959) 2024-11-10T04:38:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T04:38:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T04:38:09,080 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8 2024-11-10T04:38:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742143_1319 (size=53) 2024-11-10T04:38:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T04:38:09,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:38:09,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6d816208b10c05c54c05576e9800ef1c, disabling compactions & flushes 2024-11-10T04:38:09,486 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:09,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:09,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. after waiting 0 ms 2024-11-10T04:38:09,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:09,486 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:09,486 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6d816208b10c05c54c05576e9800ef1c: Waiting for close lock at 1731213489486Disabling compacts and flushes for region at 1731213489486Disabling writes for close at 1731213489486Writing region close event to WAL at 1731213489486Closed at 1731213489486 2024-11-10T04:38:09,487 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:38:09,487 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731213489487"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731213489487"}]},"ts":"1731213489487"} 2024-11-10T04:38:09,489 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:38:09,489 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:38:09,489 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213489489"}]},"ts":"1731213489489"} 2024-11-10T04:38:09,490 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T04:38:09,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, ASSIGN}] 2024-11-10T04:38:09,492 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, ASSIGN 2024-11-10T04:38:09,492 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,45693,1731213391842; forceNewPlan=false, retain=false 2024-11-10T04:38:09,643 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=6d816208b10c05c54c05576e9800ef1c, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:09,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, ASSIGN because future has completed 2024-11-10T04:38:09,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:38:09,799 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:09,799 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:38:09,800 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,800 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:38:09,800 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7794): checking encryption for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,800 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7797): checking classloading for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,801 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,802 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d816208b10c05c54c05576e9800ef1c columnFamilyName A 2024-11-10T04:38:09,802 DEBUG [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:09,803 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(327): Store=6d816208b10c05c54c05576e9800ef1c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:38:09,803 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,804 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d816208b10c05c54c05576e9800ef1c columnFamilyName B 2024-11-10T04:38:09,804 DEBUG [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:09,804 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(327): Store=6d816208b10c05c54c05576e9800ef1c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:38:09,804 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,805 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d816208b10c05c54c05576e9800ef1c columnFamilyName C 2024-11-10T04:38:09,805 DEBUG [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:09,805 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(327): Store=6d816208b10c05c54c05576e9800ef1c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:38:09,805 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1038): replaying wal for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,806 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,806 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,807 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1048): stopping wal replay for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,807 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1060): Cleaning up temporary data for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,807 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:38:09,808 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1093): writing seq id for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,810 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:38:09,810 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1114): Opened 6d816208b10c05c54c05576e9800ef1c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69199158, jitterRate=0.031147807836532593}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:38:09,810 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:09,811 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1006): Region open journal for 6d816208b10c05c54c05576e9800ef1c: Running coprocessor pre-open hook at 1731213489800Writing region info on filesystem at 1731213489800Initializing all the Stores at 1731213489801 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213489801Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213489801Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213489801Cleaning up temporary data from old regions at 1731213489807 (+6 ms)Running coprocessor post-open hooks at 1731213489810 (+3 ms)Region opened successfully at 1731213489811 (+1 ms) 2024-11-10T04:38:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T04:38:09,811 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., pid=97, masterSystemTime=1731213489796 2024-11-10T04:38:09,813 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:09,813 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:09,813 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=6d816208b10c05c54c05576e9800ef1c, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:09,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:38:09,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-10T04:38:09,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 in 170 msec 2024-11-10T04:38:09,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-10T04:38:09,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, ASSIGN in 325 msec 2024-11-10T04:38:09,818 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:38:09,819 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213489819"}]},"ts":"1731213489819"} 2024-11-10T04:38:09,820 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T04:38:09,820 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:38:09,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-10T04:38:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T04:38:10,831 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:10,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:10,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:10,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:10,834 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192b375a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:10,835 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58aea8a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:10,836 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:10,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:10,838 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:38:10,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,42929,1731213391190 2024-11-10T04:38:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4425de89 2024-11-10T04:38:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:38:10,839 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:38:10,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T04:38:10,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:38:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:10,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742144_1320 (size=995) 2024-11-10T04:38:11,253 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T04:38:11,253 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T04:38:11,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T04:38:11,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, REOPEN/MOVE}] 2024-11-10T04:38:11,258 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, REOPEN/MOVE 2024-11-10T04:38:11,258 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=6d816208b10c05c54c05576e9800ef1c, regionState=CLOSING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:11,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, REOPEN/MOVE because future has completed 2024-11-10T04:38:11,260 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T04:38:11,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:38:11,412 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(122): Close 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,412 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T04:38:11,412 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1722): Closing 6d816208b10c05c54c05576e9800ef1c, disabling compactions & flushes 2024-11-10T04:38:11,412 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:11,412 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:11,412 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. after waiting 0 ms 2024-11-10T04:38:11,412 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:11,415 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T04:38:11,416 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:11,416 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1676): Region close journal for 6d816208b10c05c54c05576e9800ef1c: Waiting for close lock at 1731213491412Running coprocessor pre-close hooks at 1731213491412Disabling compacts and flushes for region at 1731213491412Disabling writes for close at 1731213491412Writing region close event to WAL at 1731213491412Running coprocessor post-close hooks at 1731213491416 (+4 ms)Closed at 1731213491416 2024-11-10T04:38:11,416 WARN [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionServer(3299): Not adding moved region record: 6d816208b10c05c54c05576e9800ef1c to self. 2024-11-10T04:38:11,417 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(157): Closed 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,418 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=6d816208b10c05c54c05576e9800ef1c, regionState=CLOSED 2024-11-10T04:38:11,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:38:11,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-10T04:38:11,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; CloseRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 in 160 msec 2024-11-10T04:38:11,421 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, REOPEN/MOVE; state=CLOSED, location=d1d9e3766cb1,45693,1731213391842; forceNewPlan=false, retain=true 2024-11-10T04:38:11,572 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=6d816208b10c05c54c05576e9800ef1c, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:11,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, REOPEN/MOVE because future has completed 2024-11-10T04:38:11,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:38:11,584 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T04:38:11,728 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:11,728 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:38:11,728 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,728 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:38:11,728 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7794): checking encryption for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,728 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7797): checking classloading for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,729 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,730 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d816208b10c05c54c05576e9800ef1c columnFamilyName A 2024-11-10T04:38:11,731 DEBUG [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:11,731 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(327): Store=6d816208b10c05c54c05576e9800ef1c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:38:11,732 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,732 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d816208b10c05c54c05576e9800ef1c columnFamilyName B 2024-11-10T04:38:11,732 DEBUG [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:11,733 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(327): Store=6d816208b10c05c54c05576e9800ef1c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:38:11,733 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,733 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d816208b10c05c54c05576e9800ef1c columnFamilyName C 2024-11-10T04:38:11,733 DEBUG [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:11,734 INFO [StoreOpener-6d816208b10c05c54c05576e9800ef1c-1 {}] regionserver.HStore(327): Store=6d816208b10c05c54c05576e9800ef1c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:38:11,734 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1038): replaying wal for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,734 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,735 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,736 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1048): stopping wal replay for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,736 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1060): Cleaning up temporary data for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,736 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:38:11,737 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1093): writing seq id for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,738 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1114): Opened 6d816208b10c05c54c05576e9800ef1c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74005641, jitterRate=0.10276998579502106}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:38:11,738 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,739 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1006): Region open journal for 6d816208b10c05c54c05576e9800ef1c: Running coprocessor pre-open hook at 1731213491728Writing region info on filesystem at 1731213491728Initializing all the Stores at 1731213491729 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213491729Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213491729Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213491729Cleaning up temporary data from old regions at 1731213491736 (+7 ms)Running coprocessor post-open hooks at 1731213491738 (+2 ms)Region opened successfully at 1731213491739 (+1 ms) 2024-11-10T04:38:11,739 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., pid=102, masterSystemTime=1731213491725 2024-11-10T04:38:11,741 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:11,741 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:11,741 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=6d816208b10c05c54c05576e9800ef1c, regionState=OPEN, openSeqNum=5, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:11,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:38:11,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=100 2024-11-10T04:38:11,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 in 170 msec 2024-11-10T04:38:11,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-10T04:38:11,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, REOPEN/MOVE in 487 msec 2024-11-10T04:38:11,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-10T04:38:11,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-10T04:38:11,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-10T04:38:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-10T04:38:11,750 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:11,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,752 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302a307, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,753 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78a66fe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,755 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf524dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,757 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a8554b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,758 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4332c5cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,760 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b48056c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,761 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6daee926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,763 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b316a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,764 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@271572be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ea24d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4063fb0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c110bfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,770 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516c232f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,771 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c15e79a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,772 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad10086, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4adb681a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62e493c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1deffedc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:11,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:11,778 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56322580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:11,779 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfe1abb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:11,780 DEBUG [Thread-1469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,780 DEBUG [Thread-1488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,780 DEBUG [Thread-1473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,780 DEBUG [Thread-1482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,780 DEBUG [Thread-1486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,781 DEBUG [Thread-1477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,782 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:11,783 DEBUG [Thread-1480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,783 DEBUG [Thread-1484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,784 DEBUG [Thread-1471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,784 DEBUG [Thread-1484 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,784 DEBUG [Thread-1475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-10T04:38:11,785 DEBUG [Thread-1484 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T04:38:11,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,785 DEBUG [Thread-1471 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:11,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,786 DEBUG [Thread-1471 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:11,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,787 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,788 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:11,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:11,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:11,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:38:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:11,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:38:11,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109a6d014541fb45e58c0088c6a278d800_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213491797/Put/seqid=0 2024-11-10T04:38:11,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742145_1321 (size=17034) 2024-11-10T04:38:11,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213551876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:11,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213551876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213551876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:11,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:11,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213551877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213551878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:11,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T04:38:11,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T04:38:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:11,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:11,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213551991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213551991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213551991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213551991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213551992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T04:38:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T04:38:12,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213552211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213552212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213552212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213552212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213552212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:12,239 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109a6d014541fb45e58c0088c6a278d800_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109a6d014541fb45e58c0088c6a278d800_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:12,240 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a24d8373ae164430b2d2667ab9cd68f7, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:12,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a24d8373ae164430b2d2667ab9cd68f7 is 175, key is test_row_0/A:col10/1731213491797/Put/seqid=0 2024-11-10T04:38:12,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742146_1322 (size=48139) 2024-11-10T04:38:12,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T04:38:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T04:38:12,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:12,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T04:38:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213552531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213552531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213552531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213552532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213552532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:12,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T04:38:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,647 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a24d8373ae164430b2d2667ab9cd68f7 2024-11-10T04:38:12,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/fb767dc07286490fa26b31a4e68d5305 is 50, key is test_row_0/B:col10/1731213491797/Put/seqid=0 2024-11-10T04:38:12,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742147_1323 (size=12001) 2024-11-10T04:38:12,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/fb767dc07286490fa26b31a4e68d5305 2024-11-10T04:38:12,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/b01d0b570b2b40aa93b98ddfce9c135a is 50, key is test_row_0/C:col10/1731213491797/Put/seqid=0 2024-11-10T04:38:12,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742148_1324 (size=12001) 2024-11-10T04:38:12,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T04:38:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T04:38:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:12,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T04:38:13,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T04:38:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:13,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213553041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213553051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213553051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213553051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:13,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213553051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:13,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/b01d0b570b2b40aa93b98ddfce9c135a 2024-11-10T04:38:13,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a24d8373ae164430b2d2667ab9cd68f7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a24d8373ae164430b2d2667ab9cd68f7 2024-11-10T04:38:13,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a24d8373ae164430b2d2667ab9cd68f7, entries=250, sequenceid=16, filesize=47.0 K 2024-11-10T04:38:13,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/fb767dc07286490fa26b31a4e68d5305 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/fb767dc07286490fa26b31a4e68d5305 2024-11-10T04:38:13,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/fb767dc07286490fa26b31a4e68d5305, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T04:38:13,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/b01d0b570b2b40aa93b98ddfce9c135a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/b01d0b570b2b40aa93b98ddfce9c135a 2024-11-10T04:38:13,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/b01d0b570b2b40aa93b98ddfce9c135a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T04:38:13,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6d816208b10c05c54c05576e9800ef1c in 1320ms, sequenceid=16, compaction requested=false 2024-11-10T04:38:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T04:38:13,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:13,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T04:38:13,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:13,167 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T04:38:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111019f9be000874448680c4ebc51ee8c917_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213491877/Put/seqid=0 2024-11-10T04:38:13,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742149_1325 (size=12154) 2024-11-10T04:38:13,434 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T04:38:13,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:13,584 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111019f9be000874448680c4ebc51ee8c917_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111019f9be000874448680c4ebc51ee8c917_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/9d2c010923d14d2386bea7506267b6c7, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:13,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/9d2c010923d14d2386bea7506267b6c7 is 175, key is test_row_0/A:col10/1731213491877/Put/seqid=0 2024-11-10T04:38:13,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742150_1326 (size=30955) 2024-11-10T04:38:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T04:38:13,994 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/9d2c010923d14d2386bea7506267b6c7 2024-11-10T04:38:14,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/f826d39305b64b4db542ddff0255cf2d is 50, key is test_row_0/B:col10/1731213491877/Put/seqid=0 2024-11-10T04:38:14,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742151_1327 (size=12001) 2024-11-10T04:38:14,007 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/f826d39305b64b4db542ddff0255cf2d 2024-11-10T04:38:14,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/0487099194db48ed9ec5417658ba69eb is 50, key is test_row_0/C:col10/1731213491877/Put/seqid=0 2024-11-10T04:38:14,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742152_1328 (size=12001) 2024-11-10T04:38:14,018 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/0487099194db48ed9ec5417658ba69eb 2024-11-10T04:38:14,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/9d2c010923d14d2386bea7506267b6c7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/9d2c010923d14d2386bea7506267b6c7 2024-11-10T04:38:14,028 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/9d2c010923d14d2386bea7506267b6c7, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T04:38:14,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/f826d39305b64b4db542ddff0255cf2d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/f826d39305b64b4db542ddff0255cf2d 2024-11-10T04:38:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,032 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/f826d39305b64b4db542ddff0255cf2d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T04:38:14,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/0487099194db48ed9ec5417658ba69eb as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/0487099194db48ed9ec5417658ba69eb 2024-11-10T04:38:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,036 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/0487099194db48ed9ec5417658ba69eb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T04:38:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,037 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 6d816208b10c05c54c05576e9800ef1c in 871ms, sequenceid=40, compaction requested=false 2024-11-10T04:38:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:14,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-10T04:38:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-10T04:38:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-10T04:38:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2510 sec 2024-11-10T04:38:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.2590 sec 2024-11-10T04:38:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,164 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:14,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:38:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110490482d8e9df4c6e9d2a99eb76568140_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213494158/Put/seqid=0 2024-11-10T04:38:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742154_1330 (size=24358) 2024-11-10T04:38:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213554215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213554216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213554217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213554227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213554227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213554331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213554331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213554331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213554332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213554332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213554541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213554551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213554551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213554552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213554552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:14,581 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110490482d8e9df4c6e9d2a99eb76568140_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110490482d8e9df4c6e9d2a99eb76568140_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:14,582 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/718989cd125944c19585a90a5d44911f, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:14,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/718989cd125944c19585a90a5d44911f is 175, key is test_row_0/A:col10/1731213494158/Put/seqid=0 2024-11-10T04:38:14,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742153_1329 (size=73995) 2024-11-10T04:38:14,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213554851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213554871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213554871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213554871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213554871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:14,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/718989cd125944c19585a90a5d44911f 2024-11-10T04:38:14,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/cae189d5fd6846a29789d8ebbc14e70e is 50, key is test_row_0/B:col10/1731213494158/Put/seqid=0 2024-11-10T04:38:14,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742155_1331 (size=12001) 2024-11-10T04:38:15,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213555361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:15,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/cae189d5fd6846a29789d8ebbc14e70e 2024-11-10T04:38:15,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213555391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:15,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213555391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:15,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213555392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:15,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213555392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:15,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/252afd3c1a184d6cba4fb9d052dcf6c9 is 50, key is test_row_0/C:col10/1731213494158/Put/seqid=0 2024-11-10T04:38:15,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742156_1332 (size=12001) 2024-11-10T04:38:15,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/252afd3c1a184d6cba4fb9d052dcf6c9 2024-11-10T04:38:15,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/718989cd125944c19585a90a5d44911f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/718989cd125944c19585a90a5d44911f 2024-11-10T04:38:15,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/718989cd125944c19585a90a5d44911f, entries=400, sequenceid=51, filesize=72.3 K 2024-11-10T04:38:15,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/cae189d5fd6846a29789d8ebbc14e70e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cae189d5fd6846a29789d8ebbc14e70e 2024-11-10T04:38:15,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cae189d5fd6846a29789d8ebbc14e70e, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T04:38:15,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/252afd3c1a184d6cba4fb9d052dcf6c9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/252afd3c1a184d6cba4fb9d052dcf6c9 2024-11-10T04:38:15,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/252afd3c1a184d6cba4fb9d052dcf6c9, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T04:38:15,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6d816208b10c05c54c05576e9800ef1c in 1661ms, sequenceid=51, compaction requested=true 2024-11-10T04:38:15,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:15,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:15,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:15,825 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:15,825 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:15,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:15,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:15,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:15,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:15,826 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 153089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:15,826 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/A is initiating minor compaction (all files) 2024-11-10T04:38:15,827 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/A in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:15,827 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a24d8373ae164430b2d2667ab9cd68f7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/9d2c010923d14d2386bea7506267b6c7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/718989cd125944c19585a90a5d44911f] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=149.5 K 2024-11-10T04:38:15,827 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:15,827 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a24d8373ae164430b2d2667ab9cd68f7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/9d2c010923d14d2386bea7506267b6c7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/718989cd125944c19585a90a5d44911f] 2024-11-10T04:38:15,827 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:15,827 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/B is initiating minor compaction (all files) 2024-11-10T04:38:15,827 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a24d8373ae164430b2d2667ab9cd68f7, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731213491792 2024-11-10T04:38:15,827 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/B in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:15,827 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/fb767dc07286490fa26b31a4e68d5305, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/f826d39305b64b4db542ddff0255cf2d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cae189d5fd6846a29789d8ebbc14e70e] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=35.2 K 2024-11-10T04:38:15,828 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d2c010923d14d2386bea7506267b6c7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731213491874 2024-11-10T04:38:15,828 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fb767dc07286490fa26b31a4e68d5305, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731213491792 2024-11-10T04:38:15,828 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 718989cd125944c19585a90a5d44911f, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731213494123 2024-11-10T04:38:15,828 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f826d39305b64b4db542ddff0255cf2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731213491874 2024-11-10T04:38:15,829 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cae189d5fd6846a29789d8ebbc14e70e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731213494158 2024-11-10T04:38:15,834 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:15,835 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:15,835 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/41c6e31c7344424db781b55af9545287 is 50, key is test_row_0/B:col10/1731213494158/Put/seqid=0 2024-11-10T04:38:15,838 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d1e239cbd33a4511abee031e587cdd09_6d816208b10c05c54c05576e9800ef1c store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:15,840 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d1e239cbd33a4511abee031e587cdd09_6d816208b10c05c54c05576e9800ef1c, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:15,840 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d1e239cbd33a4511abee031e587cdd09_6d816208b10c05c54c05576e9800ef1c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:15,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742157_1333 (size=12104) 2024-11-10T04:38:15,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742158_1334 (size=4469) 2024-11-10T04:38:15,854 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#A#compaction#286 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:15,855 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a120995283914269999be978581e9cf3 is 175, key is test_row_0/A:col10/1731213494158/Put/seqid=0 2024-11-10T04:38:15,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742159_1335 (size=31058) 2024-11-10T04:38:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T04:38:15,951 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:15,951 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-10T04:38:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T04:38:15,953 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:15,953 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:15,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T04:38:16,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T04:38:16,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:16,106 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:38:16,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e19cde29157b4b43b8bf7653f94c49a2_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213494202/Put/seqid=0 2024-11-10T04:38:16,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742160_1336 (size=12154) 2024-11-10T04:38:16,258 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/41c6e31c7344424db781b55af9545287 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/41c6e31c7344424db781b55af9545287 2024-11-10T04:38:16,263 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a120995283914269999be978581e9cf3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a120995283914269999be978581e9cf3 2024-11-10T04:38:16,263 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/B of 6d816208b10c05c54c05576e9800ef1c into 41c6e31c7344424db781b55af9545287(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:16,263 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:16,263 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/B, priority=13, startTime=1731213495825; duration=0sec 2024-11-10T04:38:16,263 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:16,263 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:B 2024-11-10T04:38:16,263 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:16,264 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:16,265 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/C is initiating minor compaction (all files) 2024-11-10T04:38:16,265 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/C in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:16,265 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/b01d0b570b2b40aa93b98ddfce9c135a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/0487099194db48ed9ec5417658ba69eb, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/252afd3c1a184d6cba4fb9d052dcf6c9] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=35.2 K 2024-11-10T04:38:16,265 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b01d0b570b2b40aa93b98ddfce9c135a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731213491792 2024-11-10T04:38:16,265 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0487099194db48ed9ec5417658ba69eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731213491874 2024-11-10T04:38:16,266 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 252afd3c1a184d6cba4fb9d052dcf6c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731213494158 2024-11-10T04:38:16,269 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/A of 6d816208b10c05c54c05576e9800ef1c into a120995283914269999be978581e9cf3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:16,269 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:16,269 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/A, priority=13, startTime=1731213495825; duration=0sec 2024-11-10T04:38:16,269 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:16,269 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:A 2024-11-10T04:38:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T04:38:16,274 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#C#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:16,274 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/82aa5c3859d248d7a2848646b3ecc889 is 50, key is test_row_0/C:col10/1731213494158/Put/seqid=0 2024-11-10T04:38:16,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742161_1337 (size=12104) 2024-11-10T04:38:16,291 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/82aa5c3859d248d7a2848646b3ecc889 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/82aa5c3859d248d7a2848646b3ecc889 2024-11-10T04:38:16,296 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/C of 6d816208b10c05c54c05576e9800ef1c into 82aa5c3859d248d7a2848646b3ecc889(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:16,296 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:16,296 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/C, priority=13, startTime=1731213495825; duration=0sec 2024-11-10T04:38:16,296 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:16,296 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:C 2024-11-10T04:38:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:16,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:16,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213556413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213556414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213556415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213556416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213556417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:16,524 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e19cde29157b4b43b8bf7653f94c49a2_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e19cde29157b4b43b8bf7653f94c49a2_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:16,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/88d433a3de244f9b858e2d34815bf7d4, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:16,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/88d433a3de244f9b858e2d34815bf7d4 is 175, key is test_row_0/A:col10/1731213494202/Put/seqid=0 2024-11-10T04:38:16,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742162_1338 (size=30955) 2024-11-10T04:38:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213556521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213556532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213556532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213556532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213556532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T04:38:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213556741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213556751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213556751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213556751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213556751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:16,931 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/88d433a3de244f9b858e2d34815bf7d4 2024-11-10T04:38:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/e8706906ff714b649658bc973153d571 is 50, key is test_row_0/B:col10/1731213494202/Put/seqid=0 2024-11-10T04:38:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742163_1339 (size=12001) 2024-11-10T04:38:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213557061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213557071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213557071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213557071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213557071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T04:38:17,342 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/e8706906ff714b649658bc973153d571 2024-11-10T04:38:17,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/a05e589c53c940fd8ca4f841990d5892 is 50, key is test_row_0/C:col10/1731213494202/Put/seqid=0 2024-11-10T04:38:17,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742164_1340 (size=12001) 2024-11-10T04:38:17,361 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/a05e589c53c940fd8ca4f841990d5892 2024-11-10T04:38:17,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/88d433a3de244f9b858e2d34815bf7d4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/88d433a3de244f9b858e2d34815bf7d4 2024-11-10T04:38:17,370 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/88d433a3de244f9b858e2d34815bf7d4, entries=150, sequenceid=77, filesize=30.2 K 2024-11-10T04:38:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/e8706906ff714b649658bc973153d571 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/e8706906ff714b649658bc973153d571 2024-11-10T04:38:17,375 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/e8706906ff714b649658bc973153d571, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T04:38:17,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/a05e589c53c940fd8ca4f841990d5892 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a05e589c53c940fd8ca4f841990d5892 2024-11-10T04:38:17,380 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a05e589c53c940fd8ca4f841990d5892, entries=150, sequenceid=77, filesize=11.7 K 2024-11-10T04:38:17,382 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6d816208b10c05c54c05576e9800ef1c in 1275ms, sequenceid=77, compaction requested=false 2024-11-10T04:38:17,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:17,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:17,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-10T04:38:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-10T04:38:17,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-10T04:38:17,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-10T04:38:17,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4350 sec 2024-11-10T04:38:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:17,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:17,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:38:17,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109c7ba6ea8d5b4709a15a51618da94c85_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213496416/Put/seqid=0 2024-11-10T04:38:17,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742165_1341 (size=14594) 2024-11-10T04:38:17,589 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:17,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109c7ba6ea8d5b4709a15a51618da94c85_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109c7ba6ea8d5b4709a15a51618da94c85_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:17,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/7f41e231ca68428b9de2483320953139, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:17,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/7f41e231ca68428b9de2483320953139 is 175, key is test_row_0/A:col10/1731213496416/Put/seqid=0 2024-11-10T04:38:17,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742166_1342 (size=39549) 2024-11-10T04:38:17,599 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/7f41e231ca68428b9de2483320953139 2024-11-10T04:38:17,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/a144a436056343c3920023a8b2a85a75 is 50, key is test_row_0/B:col10/1731213496416/Put/seqid=0 2024-11-10T04:38:17,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742167_1343 (size=12001) 2024-11-10T04:38:17,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213557635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213557647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213557648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213557649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213557650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213557761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213557761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213557771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213557771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213557772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213557971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213557971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213557991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213557992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:17,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213557992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/a144a436056343c3920023a8b2a85a75 2024-11-10T04:38:18,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/8157a8ee178047318bbcdb0911ecffb5 is 50, key is test_row_0/C:col10/1731213496416/Put/seqid=0 2024-11-10T04:38:18,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742168_1344 (size=12001) 2024-11-10T04:38:18,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/8157a8ee178047318bbcdb0911ecffb5 2024-11-10T04:38:18,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/7f41e231ca68428b9de2483320953139 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/7f41e231ca68428b9de2483320953139 2024-11-10T04:38:18,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/7f41e231ca68428b9de2483320953139, entries=200, sequenceid=91, filesize=38.6 K 2024-11-10T04:38:18,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/a144a436056343c3920023a8b2a85a75 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/a144a436056343c3920023a8b2a85a75 2024-11-10T04:38:18,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/a144a436056343c3920023a8b2a85a75, entries=150, sequenceid=91, filesize=11.7 K 2024-11-10T04:38:18,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/8157a8ee178047318bbcdb0911ecffb5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8157a8ee178047318bbcdb0911ecffb5 2024-11-10T04:38:18,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8157a8ee178047318bbcdb0911ecffb5, entries=150, sequenceid=91, filesize=11.7 K 2024-11-10T04:38:18,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6d816208b10c05c54c05576e9800ef1c in 461ms, sequenceid=91, compaction requested=true 2024-11-10T04:38:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:18,038 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:18,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:18,038 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:18,040 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:18,040 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:18,040 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/A is initiating minor compaction (all files) 2024-11-10T04:38:18,040 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/B is initiating minor compaction (all files) 2024-11-10T04:38:18,040 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/B in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:18,040 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/A in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:18,040 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/41c6e31c7344424db781b55af9545287, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/e8706906ff714b649658bc973153d571, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/a144a436056343c3920023a8b2a85a75] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=35.3 K 2024-11-10T04:38:18,040 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a120995283914269999be978581e9cf3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/88d433a3de244f9b858e2d34815bf7d4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/7f41e231ca68428b9de2483320953139] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=99.2 K 2024-11-10T04:38:18,040 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:18,040 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a120995283914269999be978581e9cf3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/88d433a3de244f9b858e2d34815bf7d4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/7f41e231ca68428b9de2483320953139] 2024-11-10T04:38:18,041 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c6e31c7344424db781b55af9545287, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731213494158 2024-11-10T04:38:18,041 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a120995283914269999be978581e9cf3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731213494158 2024-11-10T04:38:18,041 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e8706906ff714b649658bc973153d571, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731213494202 2024-11-10T04:38:18,041 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d433a3de244f9b858e2d34815bf7d4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731213494202 2024-11-10T04:38:18,041 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a144a436056343c3920023a8b2a85a75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731213496415 2024-11-10T04:38:18,041 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f41e231ca68428b9de2483320953139, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731213496414 2024-11-10T04:38:18,052 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:18,052 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/916bc9523b914bd99caeb94bcc6d868b is 50, key is test_row_0/B:col10/1731213496416/Put/seqid=0 2024-11-10T04:38:18,054 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:18,057 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411103c4109cbaaf9417b8b94699cc9a2bb18_6d816208b10c05c54c05576e9800ef1c store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:18,058 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411103c4109cbaaf9417b8b94699cc9a2bb18_6d816208b10c05c54c05576e9800ef1c, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:18,058 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103c4109cbaaf9417b8b94699cc9a2bb18_6d816208b10c05c54c05576e9800ef1c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:18,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742169_1345 (size=12207) 2024-11-10T04:38:18,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742170_1346 (size=4469) 2024-11-10T04:38:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T04:38:18,101 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:18,101 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-10T04:38:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T04:38:18,103 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:18,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:18,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T04:38:18,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T04:38:18,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:18,257 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:38:18,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102c0f895a5b014e55a61a094d778a87b9_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213497648/Put/seqid=0 2024-11-10T04:38:18,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742171_1347 (size=12154) 2024-11-10T04:38:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:18,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213558312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213558312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213558313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213558313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213558314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T04:38:18,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213558431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213558431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213558431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213558431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213558431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,474 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#A#compaction#295 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:18,475 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/c1365188d81242c599be0823428f1e37 is 175, key is test_row_0/A:col10/1731213496416/Put/seqid=0 2024-11-10T04:38:18,476 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/916bc9523b914bd99caeb94bcc6d868b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/916bc9523b914bd99caeb94bcc6d868b 2024-11-10T04:38:18,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742172_1348 (size=31161) 2024-11-10T04:38:18,481 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/B of 6d816208b10c05c54c05576e9800ef1c into 916bc9523b914bd99caeb94bcc6d868b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:18,481 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:18,481 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/B, priority=13, startTime=1731213498038; duration=0sec 2024-11-10T04:38:18,481 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:18,481 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:B 2024-11-10T04:38:18,481 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:18,482 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:18,482 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/C is initiating minor compaction (all files) 2024-11-10T04:38:18,482 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/C in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:18,482 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/82aa5c3859d248d7a2848646b3ecc889, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a05e589c53c940fd8ca4f841990d5892, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8157a8ee178047318bbcdb0911ecffb5] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=35.3 K 2024-11-10T04:38:18,483 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 82aa5c3859d248d7a2848646b3ecc889, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731213494158 2024-11-10T04:38:18,483 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a05e589c53c940fd8ca4f841990d5892, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731213494202 2024-11-10T04:38:18,484 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8157a8ee178047318bbcdb0911ecffb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731213496415 2024-11-10T04:38:18,488 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/c1365188d81242c599be0823428f1e37 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/c1365188d81242c599be0823428f1e37 2024-11-10T04:38:18,494 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/A of 6d816208b10c05c54c05576e9800ef1c into c1365188d81242c599be0823428f1e37(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:18,494 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:18,494 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/A, priority=13, startTime=1731213498038; duration=0sec 2024-11-10T04:38:18,494 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:18,494 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:A 2024-11-10T04:38:18,498 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:18,498 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/6a57247ac01f4b948fee97365852051e is 50, key is test_row_0/C:col10/1731213496416/Put/seqid=0 2024-11-10T04:38:18,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742173_1349 (size=12207) 2024-11-10T04:38:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213558641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213558641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213558641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213558641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213558641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:18,673 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102c0f895a5b014e55a61a094d778a87b9_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102c0f895a5b014e55a61a094d778a87b9_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:18,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f2493042381b4d35984e13b0ffc5052e, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:18,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f2493042381b4d35984e13b0ffc5052e is 175, key is test_row_0/A:col10/1731213497648/Put/seqid=0 2024-11-10T04:38:18,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742174_1350 (size=30955) 2024-11-10T04:38:18,679 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f2493042381b4d35984e13b0ffc5052e 2024-11-10T04:38:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/1b6528323eb34a979ed3a2d8008799a7 is 50, key is test_row_0/B:col10/1731213497648/Put/seqid=0 2024-11-10T04:38:18,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742175_1351 (size=12001) 2024-11-10T04:38:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T04:38:18,917 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/6a57247ac01f4b948fee97365852051e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/6a57247ac01f4b948fee97365852051e 2024-11-10T04:38:18,922 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/C of 6d816208b10c05c54c05576e9800ef1c into 6a57247ac01f4b948fee97365852051e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:18,922 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:18,922 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/C, priority=13, startTime=1731213498038; duration=0sec 2024-11-10T04:38:18,922 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:18,922 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:C 2024-11-10T04:38:18,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213558961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213558961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213558961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213558962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213558962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:19,092 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/1b6528323eb34a979ed3a2d8008799a7 2024-11-10T04:38:19,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/f2f5f13c36264086ab5b90a1cc27107d is 50, key is test_row_0/C:col10/1731213497648/Put/seqid=0 2024-11-10T04:38:19,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742176_1352 (size=12001) 2024-11-10T04:38:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T04:38:19,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213559481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:19,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213559482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:19,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213559482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:19,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213559482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213559482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:19,504 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/f2f5f13c36264086ab5b90a1cc27107d 2024-11-10T04:38:19,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f2493042381b4d35984e13b0ffc5052e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2493042381b4d35984e13b0ffc5052e 2024-11-10T04:38:19,512 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2493042381b4d35984e13b0ffc5052e, entries=150, sequenceid=117, filesize=30.2 K 2024-11-10T04:38:19,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/1b6528323eb34a979ed3a2d8008799a7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/1b6528323eb34a979ed3a2d8008799a7 2024-11-10T04:38:19,516 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/1b6528323eb34a979ed3a2d8008799a7, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T04:38:19,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/f2f5f13c36264086ab5b90a1cc27107d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/f2f5f13c36264086ab5b90a1cc27107d 2024-11-10T04:38:19,521 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/f2f5f13c36264086ab5b90a1cc27107d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T04:38:19,522 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6d816208b10c05c54c05576e9800ef1c in 1265ms, sequenceid=117, compaction requested=false 2024-11-10T04:38:19,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:19,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:19,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-10T04:38:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-10T04:38:19,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T04:38:19,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-11-10T04:38:19,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4250 sec 2024-11-10T04:38:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T04:38:20,262 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:20,262 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-10T04:38:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T04:38:20,264 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:20,264 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:20,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T04:38:20,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T04:38:20,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:20,418 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T04:38:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109cc6b9f9533249a7b671fba0ccb6d9f2_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213498299/Put/seqid=0 2024-11-10T04:38:20,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742177_1353 (size=12154) 2024-11-10T04:38:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:20,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:20,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213560546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213560546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213560547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213560548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213560549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T04:38:20,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213560661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213560661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213560671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213560672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213560672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:20,833 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109cc6b9f9533249a7b671fba0ccb6d9f2_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109cc6b9f9533249a7b671fba0ccb6d9f2_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:20,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f2c2a51562a64ff79815127f70d852d5, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:20,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f2c2a51562a64ff79815127f70d852d5 is 175, key is test_row_0/A:col10/1731213498299/Put/seqid=0 2024-11-10T04:38:20,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742178_1354 (size=30955) 2024-11-10T04:38:20,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213560881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213560881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T04:38:20,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213560891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213560891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213560891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213561191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213561201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213561211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213561212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213561212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,238 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f2c2a51562a64ff79815127f70d852d5 2024-11-10T04:38:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/dcfa5bd133eb498782fa960a798d78b3 is 50, key is test_row_0/B:col10/1731213498299/Put/seqid=0 2024-11-10T04:38:21,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742179_1355 (size=12001) 2024-11-10T04:38:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T04:38:21,584 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T04:38:21,584 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T04:38:21,650 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/dcfa5bd133eb498782fa960a798d78b3 2024-11-10T04:38:21,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/cd2ae10d62c74254844748663f951336 is 50, key is test_row_0/C:col10/1731213498299/Put/seqid=0 2024-11-10T04:38:21,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742180_1356 (size=12001) 2024-11-10T04:38:21,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213561711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213561721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213561731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213561731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213561731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,060 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/cd2ae10d62c74254844748663f951336 2024-11-10T04:38:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f2c2a51562a64ff79815127f70d852d5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2c2a51562a64ff79815127f70d852d5 2024-11-10T04:38:22,068 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2c2a51562a64ff79815127f70d852d5, entries=150, sequenceid=130, filesize=30.2 K 2024-11-10T04:38:22,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/dcfa5bd133eb498782fa960a798d78b3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/dcfa5bd133eb498782fa960a798d78b3 2024-11-10T04:38:22,073 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/dcfa5bd133eb498782fa960a798d78b3, entries=150, sequenceid=130, filesize=11.7 K 2024-11-10T04:38:22,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/cd2ae10d62c74254844748663f951336 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/cd2ae10d62c74254844748663f951336 2024-11-10T04:38:22,078 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/cd2ae10d62c74254844748663f951336, entries=150, sequenceid=130, filesize=11.7 K 2024-11-10T04:38:22,078 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 6d816208b10c05c54c05576e9800ef1c in 1661ms, sequenceid=130, compaction requested=true 2024-11-10T04:38:22,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:22,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:22,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-10T04:38:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-10T04:38:22,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T04:38:22,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-10T04:38:22,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8200 sec 2024-11-10T04:38:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T04:38:22,411 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:22,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-10T04:38:22,413 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T04:38:22,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:22,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T04:38:22,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T04:38:22,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:22,566 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T04:38:22,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103a2d9bbd0c9543a3b0c1d30082b9b56d_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213500547/Put/seqid=0 2024-11-10T04:38:22,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742181_1357 (size=12304) 2024-11-10T04:38:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T04:38:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:22,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:22,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213562748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213562749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:22,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:22,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213562750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:22,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:22,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213562751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:22,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213562751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,762 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 6d816208b10c05c54c05576e9800ef1c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:38:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213562861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:22,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213562871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213562872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:22,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213562872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:22,982 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103a2d9bbd0c9543a3b0c1d30082b9b56d_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103a2d9bbd0c9543a3b0c1d30082b9b56d_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/47597b20710f4f4dba14de7583d8f1d6, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/47597b20710f4f4dba14de7583d8f1d6 is 175, key is test_row_0/A:col10/1731213500547/Put/seqid=0 2024-11-10T04:38:22,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742182_1358 (size=31105) 2024-11-10T04:38:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T04:38:23,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213563081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:23,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213563091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213563091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:23,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213563091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:23,388 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/47597b20710f4f4dba14de7583d8f1d6 2024-11-10T04:38:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/97c00acc49514b4ea83621c06f9a835a is 50, key is test_row_0/B:col10/1731213500547/Put/seqid=0 2024-11-10T04:38:23,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742183_1359 (size=12151) 2024-11-10T04:38:23,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213563401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:23,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213563411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:23,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213563411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:23,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213563412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T04:38:23,800 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/97c00acc49514b4ea83621c06f9a835a 2024-11-10T04:38:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/78e3b55035aa44fcb596985c0256b997 is 50, key is test_row_0/C:col10/1731213500547/Put/seqid=0 2024-11-10T04:38:23,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742184_1360 (size=12151) 2024-11-10T04:38:23,814 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/78e3b55035aa44fcb596985c0256b997 2024-11-10T04:38:23,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/47597b20710f4f4dba14de7583d8f1d6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/47597b20710f4f4dba14de7583d8f1d6 2024-11-10T04:38:23,828 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/47597b20710f4f4dba14de7583d8f1d6, entries=150, sequenceid=157, filesize=30.4 K 2024-11-10T04:38:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/97c00acc49514b4ea83621c06f9a835a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/97c00acc49514b4ea83621c06f9a835a 2024-11-10T04:38:23,837 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/97c00acc49514b4ea83621c06f9a835a, entries=150, sequenceid=157, filesize=11.9 K 2024-11-10T04:38:23,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/78e3b55035aa44fcb596985c0256b997 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/78e3b55035aa44fcb596985c0256b997 2024-11-10T04:38:23,843 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/78e3b55035aa44fcb596985c0256b997, entries=150, sequenceid=157, filesize=11.9 K 2024-11-10T04:38:23,849 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 6d816208b10c05c54c05576e9800ef1c in 1283ms, sequenceid=157, compaction requested=true 2024-11-10T04:38:23,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:23,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:23,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-10T04:38:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-10T04:38:23,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-10T04:38:23,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4390 sec 2024-11-10T04:38:23,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.4440 sec 2024-11-10T04:38:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:23,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:38:23,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108cd3c8ab2ff245dcbc1eb16993ebb4c8_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213503915/Put/seqid=0 2024-11-10T04:38:23,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742185_1361 (size=14794) 2024-11-10T04:38:23,953 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:23,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108cd3c8ab2ff245dcbc1eb16993ebb4c8_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108cd3c8ab2ff245dcbc1eb16993ebb4c8_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:23,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/78831fd46f304b448de38737aeddef8a, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:23,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/78831fd46f304b448de38737aeddef8a is 175, key is test_row_0/A:col10/1731213503915/Put/seqid=0 2024-11-10T04:38:23,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742186_1362 (size=39749) 2024-11-10T04:38:24,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213564001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213564001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213564002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213564003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213564111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213564121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213564122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213564122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213564331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213564341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213564341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213564342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/78831fd46f304b448de38737aeddef8a 2024-11-10T04:38:24,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/c0c5327782ac43e7bdf862c7a0b06c44 is 50, key is test_row_0/B:col10/1731213503915/Put/seqid=0 2024-11-10T04:38:24,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742187_1363 (size=12151) 2024-11-10T04:38:24,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/c0c5327782ac43e7bdf862c7a0b06c44 2024-11-10T04:38:24,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/572c48ba49194c6291bf6fdcc9f591f0 is 50, key is test_row_0/C:col10/1731213503915/Put/seqid=0 2024-11-10T04:38:24,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742188_1364 (size=12151) 2024-11-10T04:38:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T04:38:24,562 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:24,562 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-10T04:38:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T04:38:24,564 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:24,565 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:24,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:24,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213564651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213564651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213564651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213564661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T04:38:24,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T04:38:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:24,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:24,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213564771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:24,781 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 6d816208b10c05c54c05576e9800ef1c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:38:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:24,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/572c48ba49194c6291bf6fdcc9f591f0 2024-11-10T04:38:24,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/78831fd46f304b448de38737aeddef8a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/78831fd46f304b448de38737aeddef8a 2024-11-10T04:38:24,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/78831fd46f304b448de38737aeddef8a, entries=200, sequenceid=168, filesize=38.8 K 2024-11-10T04:38:24,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/c0c5327782ac43e7bdf862c7a0b06c44 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c0c5327782ac43e7bdf862c7a0b06c44 2024-11-10T04:38:24,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c0c5327782ac43e7bdf862c7a0b06c44, entries=150, sequenceid=168, filesize=11.9 K 2024-11-10T04:38:24,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/572c48ba49194c6291bf6fdcc9f591f0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/572c48ba49194c6291bf6fdcc9f591f0 2024-11-10T04:38:24,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/572c48ba49194c6291bf6fdcc9f591f0, entries=150, sequenceid=168, filesize=11.9 K 2024-11-10T04:38:24,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6d816208b10c05c54c05576e9800ef1c in 888ms, sequenceid=168, compaction requested=true 2024-11-10T04:38:24,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:24,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:24,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:24,813 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T04:38:24,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:24,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:24,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:24,813 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T04:38:24,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:24,814 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163925 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T04:38:24,814 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T04:38:24,814 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/A is initiating minor compaction (all files) 2024-11-10T04:38:24,814 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/B is initiating minor compaction (all files) 2024-11-10T04:38:24,814 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/A in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:24,814 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/B in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:24,814 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/c1365188d81242c599be0823428f1e37, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2493042381b4d35984e13b0ffc5052e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2c2a51562a64ff79815127f70d852d5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/47597b20710f4f4dba14de7583d8f1d6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/78831fd46f304b448de38737aeddef8a] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=160.1 K 2024-11-10T04:38:24,814 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/916bc9523b914bd99caeb94bcc6d868b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/1b6528323eb34a979ed3a2d8008799a7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/dcfa5bd133eb498782fa960a798d78b3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/97c00acc49514b4ea83621c06f9a835a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c0c5327782ac43e7bdf862c7a0b06c44] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=59.1 K 2024-11-10T04:38:24,814 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:24,814 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/c1365188d81242c599be0823428f1e37, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2493042381b4d35984e13b0ffc5052e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2c2a51562a64ff79815127f70d852d5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/47597b20710f4f4dba14de7583d8f1d6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/78831fd46f304b448de38737aeddef8a] 2024-11-10T04:38:24,815 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 916bc9523b914bd99caeb94bcc6d868b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731213496415 2024-11-10T04:38:24,815 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1365188d81242c599be0823428f1e37, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731213496415 2024-11-10T04:38:24,815 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b6528323eb34a979ed3a2d8008799a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731213497633 2024-11-10T04:38:24,815 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2493042381b4d35984e13b0ffc5052e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731213497633 2024-11-10T04:38:24,815 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting dcfa5bd133eb498782fa960a798d78b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731213498299 2024-11-10T04:38:24,815 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2c2a51562a64ff79815127f70d852d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731213498299 2024-11-10T04:38:24,816 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 97c00acc49514b4ea83621c06f9a835a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731213500545 2024-11-10T04:38:24,816 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47597b20710f4f4dba14de7583d8f1d6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731213500545 2024-11-10T04:38:24,816 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c5327782ac43e7bdf862c7a0b06c44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731213502747 2024-11-10T04:38:24,816 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78831fd46f304b448de38737aeddef8a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731213502747 2024-11-10T04:38:24,825 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:24,827 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110554483819bdd41568a7e337a493a5b11_6d816208b10c05c54c05576e9800ef1c store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:24,827 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#B#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:24,828 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/16e598a5179a42b087f37bd6d38140f1 is 50, key is test_row_0/B:col10/1731213503915/Put/seqid=0 2024-11-10T04:38:24,829 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110554483819bdd41568a7e337a493a5b11_6d816208b10c05c54c05576e9800ef1c, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:24,830 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110554483819bdd41568a7e337a493a5b11_6d816208b10c05c54c05576e9800ef1c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:24,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742189_1365 (size=12527) 2024-11-10T04:38:24,841 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/16e598a5179a42b087f37bd6d38140f1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/16e598a5179a42b087f37bd6d38140f1 2024-11-10T04:38:24,847 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/B of 6d816208b10c05c54c05576e9800ef1c into 16e598a5179a42b087f37bd6d38140f1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:24,847 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:24,847 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/B, priority=11, startTime=1731213504813; duration=0sec 2024-11-10T04:38:24,847 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:24,847 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:B 2024-11-10T04:38:24,847 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T04:38:24,849 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T04:38:24,849 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/C is initiating minor compaction (all files) 2024-11-10T04:38:24,849 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/C in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:24,849 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/6a57247ac01f4b948fee97365852051e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/f2f5f13c36264086ab5b90a1cc27107d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/cd2ae10d62c74254844748663f951336, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/78e3b55035aa44fcb596985c0256b997, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/572c48ba49194c6291bf6fdcc9f591f0] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=59.1 K 2024-11-10T04:38:24,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742190_1366 (size=4469) 2024-11-10T04:38:24,850 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a57247ac01f4b948fee97365852051e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731213496415 2024-11-10T04:38:24,850 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f5f13c36264086ab5b90a1cc27107d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731213497633 2024-11-10T04:38:24,851 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2ae10d62c74254844748663f951336, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731213498299 2024-11-10T04:38:24,852 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e3b55035aa44fcb596985c0256b997, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731213500545 2024-11-10T04:38:24,852 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 572c48ba49194c6291bf6fdcc9f591f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731213502747 2024-11-10T04:38:24,852 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#A#compaction#309 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:24,853 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/ba20f68cc848489aabd1952d19281e02 is 175, key is test_row_0/A:col10/1731213503915/Put/seqid=0 2024-11-10T04:38:24,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742191_1367 (size=31481) 2024-11-10T04:38:24,864 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:24,865 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/a9fb47a536a14d4dbb3d47633b4e7ea9 is 50, key is test_row_0/C:col10/1731213503915/Put/seqid=0 2024-11-10T04:38:24,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742192_1368 (size=12527) 2024-11-10T04:38:24,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T04:38:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:24,870 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:38:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T04:38:24,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dceb50111125490c84da2f5478005da5_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213503999/Put/seqid=0 2024-11-10T04:38:24,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742193_1369 (size=12304) 2024-11-10T04:38:24,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:24,903 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dceb50111125490c84da2f5478005da5_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dceb50111125490c84da2f5478005da5_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:24,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f6c8030ffdda41c5aa410b4ca22c648d, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:24,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f6c8030ffdda41c5aa410b4ca22c648d is 175, key is test_row_0/A:col10/1731213503999/Put/seqid=0 2024-11-10T04:38:24,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742194_1370 (size=31105) 2024-11-10T04:38:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:25,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T04:38:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213565189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213565190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213565190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213565192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,262 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/ba20f68cc848489aabd1952d19281e02 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/ba20f68cc848489aabd1952d19281e02 2024-11-10T04:38:25,266 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/A of 6d816208b10c05c54c05576e9800ef1c into ba20f68cc848489aabd1952d19281e02(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:25,266 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:25,266 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/A, priority=11, startTime=1731213504812; duration=0sec 2024-11-10T04:38:25,267 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:25,267 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:A 2024-11-10T04:38:25,273 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/a9fb47a536a14d4dbb3d47633b4e7ea9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a9fb47a536a14d4dbb3d47633b4e7ea9 2024-11-10T04:38:25,277 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/C of 6d816208b10c05c54c05576e9800ef1c into a9fb47a536a14d4dbb3d47633b4e7ea9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:25,277 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:25,277 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/C, priority=11, startTime=1731213504813; duration=0sec 2024-11-10T04:38:25,277 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:25,277 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:C 2024-11-10T04:38:25,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213565301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,315 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f6c8030ffdda41c5aa410b4ca22c648d 2024-11-10T04:38:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213565311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213565312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213565312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/2e841daf9295457aa553777225d8e629 is 50, key is test_row_0/B:col10/1731213503999/Put/seqid=0 2024-11-10T04:38:25,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742195_1371 (size=12151) 2024-11-10T04:38:25,326 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/2e841daf9295457aa553777225d8e629 2024-11-10T04:38:25,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/9cb6a905970341e3afa9b49606249695 is 50, key is test_row_0/C:col10/1731213503999/Put/seqid=0 2024-11-10T04:38:25,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742196_1372 (size=12151) 2024-11-10T04:38:25,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213565511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213565521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213565521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213565522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T04:38:25,736 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/9cb6a905970341e3afa9b49606249695 2024-11-10T04:38:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/f6c8030ffdda41c5aa410b4ca22c648d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f6c8030ffdda41c5aa410b4ca22c648d 2024-11-10T04:38:25,743 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f6c8030ffdda41c5aa410b4ca22c648d, entries=150, sequenceid=195, filesize=30.4 K 2024-11-10T04:38:25,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/2e841daf9295457aa553777225d8e629 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/2e841daf9295457aa553777225d8e629 2024-11-10T04:38:25,750 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/2e841daf9295457aa553777225d8e629, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T04:38:25,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/9cb6a905970341e3afa9b49606249695 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/9cb6a905970341e3afa9b49606249695 2024-11-10T04:38:25,757 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/9cb6a905970341e3afa9b49606249695, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T04:38:25,757 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6d816208b10c05c54c05576e9800ef1c in 887ms, sequenceid=195, compaction requested=false 2024-11-10T04:38:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-10T04:38:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-10T04:38:25,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-10T04:38:25,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1940 sec 2024-11-10T04:38:25,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.1990 sec 2024-11-10T04:38:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:25,835 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:25,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:38:25,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101db0de8e87654f4b8abea659ba2a8980_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213505190/Put/seqid=0 2024-11-10T04:38:25,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742197_1373 (size=14794) 2024-11-10T04:38:25,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213565890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213565891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213565894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:25,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213565895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213566011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213566011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213566011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213566011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213566231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213566231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213566231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213566231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,251 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:26,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101db0de8e87654f4b8abea659ba2a8980_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101db0de8e87654f4b8abea659ba2a8980_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:26,255 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/d865bd5441ca49ecb22fe78dd71337e7, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:26,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/d865bd5441ca49ecb22fe78dd71337e7 is 175, key is test_row_0/A:col10/1731213505190/Put/seqid=0 2024-11-10T04:38:26,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742198_1374 (size=39749) 2024-11-10T04:38:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213566541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213566551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213566552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213566552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:26,660 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/d865bd5441ca49ecb22fe78dd71337e7 2024-11-10T04:38:26,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/143c5a8a636c47d3b5a1cb3da52e7cba is 50, key is test_row_0/B:col10/1731213505190/Put/seqid=0 2024-11-10T04:38:26,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742199_1375 (size=12151) 2024-11-10T04:38:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T04:38:26,711 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:26,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-10T04:38:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T04:38:26,713 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:26,714 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:26,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T04:38:26,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T04:38:26,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:26,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:26,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:26,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T04:38:27,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:27,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T04:38:27,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213567061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:27,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213567061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:27,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213567061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:27,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213567061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:27,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:27,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/143c5a8a636c47d3b5a1cb3da52e7cba 2024-11-10T04:38:27,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/1f38e14cbb914451b0b0a510f97bb370 is 50, key is test_row_0/C:col10/1731213505190/Put/seqid=0 2024-11-10T04:38:27,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742200_1376 (size=12151) 2024-11-10T04:38:27,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T04:38:27,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:27,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T04:38:27,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:27,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T04:38:27,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T04:38:27,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:27,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:27,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/1f38e14cbb914451b0b0a510f97bb370 2024-11-10T04:38:27,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/d865bd5441ca49ecb22fe78dd71337e7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/d865bd5441ca49ecb22fe78dd71337e7 2024-11-10T04:38:27,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/d865bd5441ca49ecb22fe78dd71337e7, entries=200, sequenceid=208, filesize=38.8 K 2024-11-10T04:38:27,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/143c5a8a636c47d3b5a1cb3da52e7cba as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/143c5a8a636c47d3b5a1cb3da52e7cba 2024-11-10T04:38:27,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/143c5a8a636c47d3b5a1cb3da52e7cba, entries=150, sequenceid=208, filesize=11.9 K 2024-11-10T04:38:27,502 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T04:38:27,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/1f38e14cbb914451b0b0a510f97bb370 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/1f38e14cbb914451b0b0a510f97bb370 2024-11-10T04:38:27,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/1f38e14cbb914451b0b0a510f97bb370, entries=150, sequenceid=208, filesize=11.9 K 2024-11-10T04:38:27,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6d816208b10c05c54c05576e9800ef1c in 1671ms, sequenceid=208, compaction requested=true 2024-11-10T04:38:27,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:27,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:27,506 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:27,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:27,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:27,507 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:27,508 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:27,508 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:27,508 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/B is initiating minor compaction (all files) 2024-11-10T04:38:27,508 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/A is initiating minor compaction (all files) 2024-11-10T04:38:27,508 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/B in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,508 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/A in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,508 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/16e598a5179a42b087f37bd6d38140f1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/2e841daf9295457aa553777225d8e629, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/143c5a8a636c47d3b5a1cb3da52e7cba] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=36.0 K 2024-11-10T04:38:27,508 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/ba20f68cc848489aabd1952d19281e02, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f6c8030ffdda41c5aa410b4ca22c648d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/d865bd5441ca49ecb22fe78dd71337e7] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=99.9 K 2024-11-10T04:38:27,508 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,508 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/ba20f68cc848489aabd1952d19281e02, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f6c8030ffdda41c5aa410b4ca22c648d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/d865bd5441ca49ecb22fe78dd71337e7] 2024-11-10T04:38:27,508 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 16e598a5179a42b087f37bd6d38140f1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731213502747 2024-11-10T04:38:27,509 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba20f68cc848489aabd1952d19281e02, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731213502747 2024-11-10T04:38:27,509 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e841daf9295457aa553777225d8e629, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731213503999 2024-11-10T04:38:27,509 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6c8030ffdda41c5aa410b4ca22c648d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731213503999 2024-11-10T04:38:27,509 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 143c5a8a636c47d3b5a1cb3da52e7cba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731213505189 2024-11-10T04:38:27,509 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d865bd5441ca49ecb22fe78dd71337e7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731213505188 2024-11-10T04:38:27,515 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:27,516 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:27,517 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/0710778ea65e40cfb284d28798d5f64d is 50, key is test_row_0/B:col10/1731213505190/Put/seqid=0 2024-11-10T04:38:27,517 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111086bfba137caa4a6f8d691a7a484934c4_6d816208b10c05c54c05576e9800ef1c store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:27,518 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111086bfba137caa4a6f8d691a7a484934c4_6d816208b10c05c54c05576e9800ef1c, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:27,518 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111086bfba137caa4a6f8d691a7a484934c4_6d816208b10c05c54c05576e9800ef1c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:27,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742201_1377 (size=12629) 2024-11-10T04:38:27,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742202_1378 (size=4469) 2024-11-10T04:38:27,523 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#A#compaction#319 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:27,523 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/40666f2d5405423094b72c55f0b923e1 is 175, key is test_row_0/A:col10/1731213505190/Put/seqid=0 2024-11-10T04:38:27,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742203_1379 (size=31583) 2024-11-10T04:38:27,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T04:38:27,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,636 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:38:27,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b77ba567bb494c4983d2ae4234344c66_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213505889/Put/seqid=0 2024-11-10T04:38:27,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742204_1380 (size=12304) 2024-11-10T04:38:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T04:38:27,925 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/0710778ea65e40cfb284d28798d5f64d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/0710778ea65e40cfb284d28798d5f64d 2024-11-10T04:38:27,930 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/B of 6d816208b10c05c54c05576e9800ef1c into 0710778ea65e40cfb284d28798d5f64d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:27,930 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:27,930 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/B, priority=13, startTime=1731213507507; duration=0sec 2024-11-10T04:38:27,930 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:27,930 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:B 2024-11-10T04:38:27,930 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:27,931 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/40666f2d5405423094b72c55f0b923e1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/40666f2d5405423094b72c55f0b923e1 2024-11-10T04:38:27,932 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:27,932 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/C is initiating minor compaction (all files) 2024-11-10T04:38:27,932 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/C in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:27,932 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a9fb47a536a14d4dbb3d47633b4e7ea9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/9cb6a905970341e3afa9b49606249695, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/1f38e14cbb914451b0b0a510f97bb370] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=36.0 K 2024-11-10T04:38:27,932 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a9fb47a536a14d4dbb3d47633b4e7ea9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731213502747 2024-11-10T04:38:27,933 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb6a905970341e3afa9b49606249695, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731213503999 2024-11-10T04:38:27,933 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f38e14cbb914451b0b0a510f97bb370, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731213505189 2024-11-10T04:38:27,936 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/A of 6d816208b10c05c54c05576e9800ef1c into 40666f2d5405423094b72c55f0b923e1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:27,936 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:27,936 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/A, priority=13, startTime=1731213507506; duration=0sec 2024-11-10T04:38:27,936 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:27,936 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:A 2024-11-10T04:38:27,946 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#C#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:27,946 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/fd286706d0a54349af0206169dcf1e35 is 50, key is test_row_0/C:col10/1731213505190/Put/seqid=0 2024-11-10T04:38:27,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742205_1381 (size=12629) 2024-11-10T04:38:27,954 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/fd286706d0a54349af0206169dcf1e35 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/fd286706d0a54349af0206169dcf1e35 2024-11-10T04:38:27,959 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/C of 6d816208b10c05c54c05576e9800ef1c into fd286706d0a54349af0206169dcf1e35(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:27,959 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:27,959 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/C, priority=13, startTime=1731213507507; duration=0sec 2024-11-10T04:38:27,959 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:27,960 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:C 2024-11-10T04:38:28,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:28,051 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b77ba567bb494c4983d2ae4234344c66_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b77ba567bb494c4983d2ae4234344c66_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:28,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/64b6f30e746a447bacbbc0dbe739136f, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:28,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/64b6f30e746a447bacbbc0dbe739136f is 175, key is test_row_0/A:col10/1731213505889/Put/seqid=0 2024-11-10T04:38:28,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742206_1382 (size=31105) 2024-11-10T04:38:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:28,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:28,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213568099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213568100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213568101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213568102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213568211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213568221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213568222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213568222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213568431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213568441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213568441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213568441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,458 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/64b6f30e746a447bacbbc0dbe739136f 2024-11-10T04:38:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/174948c930394518a58f2d732f0c7e43 is 50, key is test_row_0/B:col10/1731213505889/Put/seqid=0 2024-11-10T04:38:28,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742207_1383 (size=12151) 2024-11-10T04:38:28,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213568751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213568761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213568761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213568762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33802 deadline: 1731213568831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:28,837 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 6d816208b10c05c54c05576e9800ef1c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:38:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T04:38:28,871 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/174948c930394518a58f2d732f0c7e43 2024-11-10T04:38:28,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/8ce8ad6a221b4a5e82ec5fe50ec94de3 is 50, key is test_row_0/C:col10/1731213505889/Put/seqid=0 2024-11-10T04:38:28,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742208_1384 (size=12151) 2024-11-10T04:38:29,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213569271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:29,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:29,283 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/8ce8ad6a221b4a5e82ec5fe50ec94de3 2024-11-10T04:38:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/64b6f30e746a447bacbbc0dbe739136f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/64b6f30e746a447bacbbc0dbe739136f 2024-11-10T04:38:29,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213569281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:29,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213569281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:29,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213569281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:29,290 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/64b6f30e746a447bacbbc0dbe739136f, entries=150, sequenceid=233, filesize=30.4 K 2024-11-10T04:38:29,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/174948c930394518a58f2d732f0c7e43 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/174948c930394518a58f2d732f0c7e43 2024-11-10T04:38:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,295 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/174948c930394518a58f2d732f0c7e43, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T04:38:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/8ce8ad6a221b4a5e82ec5fe50ec94de3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8ce8ad6a221b4a5e82ec5fe50ec94de3 2024-11-10T04:38:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,298 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8ce8ad6a221b4a5e82ec5fe50ec94de3, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T04:38:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,299 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6d816208b10c05c54c05576e9800ef1c in 1663ms, sequenceid=233, compaction requested=false 2024-11-10T04:38:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-10T04:38:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-10T04:38:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-10T04:38:29,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5880 sec 2024-11-10T04:38:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.5930 sec 2024-11-10T04:38:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,304 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:30,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:38:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,306 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:38:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102da0b8a5146148b890808c702dfc2108_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213508099/Put/seqid=0 2024-11-10T04:38:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742209_1385 (size=14794) 2024-11-10T04:38:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213570412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213570414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213570427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213570428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213570531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213570531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213570552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213570552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,731 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:30,736 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102da0b8a5146148b890808c702dfc2108_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102da0b8a5146148b890808c702dfc2108_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:30,737 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/8135f9ae13b245e5b977ce65304cdfe8, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:30,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/8135f9ae13b245e5b977ce65304cdfe8 is 175, key is test_row_0/A:col10/1731213508099/Put/seqid=0 2024-11-10T04:38:30,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742210_1386 (size=39745) 2024-11-10T04:38:30,747 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/8135f9ae13b245e5b977ce65304cdfe8 2024-11-10T04:38:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213570751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213570751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/779eca82e85b44059a806c886542aaf3 is 50, key is test_row_0/B:col10/1731213508099/Put/seqid=0 2024-11-10T04:38:30,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213570761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213570763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:30,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742211_1387 (size=9757) 2024-11-10T04:38:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T04:38:30,882 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:30,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-10T04:38:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T04:38:30,888 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:30,888 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:30,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T04:38:31,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T04:38:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:31,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213571071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213571071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213571081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213571081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/779eca82e85b44059a806c886542aaf3 2024-11-10T04:38:31,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/07b8c5d32a6e407980e62471dbdc5fa2 is 50, key is test_row_0/C:col10/1731213508099/Put/seqid=0 2024-11-10T04:38:31,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T04:38:31,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:31,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:31,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:31,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742212_1388 (size=9757) 2024-11-10T04:38:31,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/07b8c5d32a6e407980e62471dbdc5fa2 2024-11-10T04:38:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:31,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/8135f9ae13b245e5b977ce65304cdfe8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/8135f9ae13b245e5b977ce65304cdfe8 2024-11-10T04:38:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T04:38:31,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/8135f9ae13b245e5b977ce65304cdfe8, entries=200, sequenceid=249, filesize=38.8 K 2024-11-10T04:38:31,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/779eca82e85b44059a806c886542aaf3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/779eca82e85b44059a806c886542aaf3 2024-11-10T04:38:31,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/779eca82e85b44059a806c886542aaf3, entries=100, sequenceid=249, filesize=9.5 K 2024-11-10T04:38:31,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/07b8c5d32a6e407980e62471dbdc5fa2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/07b8c5d32a6e407980e62471dbdc5fa2 2024-11-10T04:38:31,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/07b8c5d32a6e407980e62471dbdc5fa2, entries=100, sequenceid=249, filesize=9.5 K 2024-11-10T04:38:31,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 6d816208b10c05c54c05576e9800ef1c in 910ms, sequenceid=249, compaction requested=true 2024-11-10T04:38:31,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:31,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:31,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:31,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:31,214 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:31,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:31,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:31,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T04:38:31,215 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:31,215 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/A is initiating minor compaction (all files) 2024-11-10T04:38:31,215 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/A in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,216 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:31,216 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/40666f2d5405423094b72c55f0b923e1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/64b6f30e746a447bacbbc0dbe739136f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/8135f9ae13b245e5b977ce65304cdfe8] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=100.0 K 2024-11-10T04:38:31,216 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,216 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/40666f2d5405423094b72c55f0b923e1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/64b6f30e746a447bacbbc0dbe739136f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/8135f9ae13b245e5b977ce65304cdfe8] 2024-11-10T04:38:31,219 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40666f2d5405423094b72c55f0b923e1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731213505189 2024-11-10T04:38:31,219 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:31,219 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/B is initiating minor compaction (all files) 2024-11-10T04:38:31,219 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64b6f30e746a447bacbbc0dbe739136f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731213505887 2024-11-10T04:38:31,219 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/B in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,219 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/0710778ea65e40cfb284d28798d5f64d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/174948c930394518a58f2d732f0c7e43, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/779eca82e85b44059a806c886542aaf3] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=33.7 K 2024-11-10T04:38:31,219 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8135f9ae13b245e5b977ce65304cdfe8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213508099 2024-11-10T04:38:31,219 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0710778ea65e40cfb284d28798d5f64d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731213505189 2024-11-10T04:38:31,220 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 174948c930394518a58f2d732f0c7e43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731213505887 2024-11-10T04:38:31,220 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 779eca82e85b44059a806c886542aaf3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213508099 2024-11-10T04:38:31,227 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:31,228 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#B#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:31,229 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411102d08c2b6a8984c9899bee3b6bfb1bb93_6d816208b10c05c54c05576e9800ef1c store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:31,229 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/c51feb96b06340dd8b4aff37971c090c is 50, key is test_row_0/B:col10/1731213508099/Put/seqid=0 2024-11-10T04:38:31,233 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411102d08c2b6a8984c9899bee3b6bfb1bb93_6d816208b10c05c54c05576e9800ef1c, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:31,233 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102d08c2b6a8984c9899bee3b6bfb1bb93_6d816208b10c05c54c05576e9800ef1c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:31,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742213_1389 (size=12731) 2024-11-10T04:38:31,246 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/c51feb96b06340dd8b4aff37971c090c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c51feb96b06340dd8b4aff37971c090c 2024-11-10T04:38:31,250 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/B of 6d816208b10c05c54c05576e9800ef1c into c51feb96b06340dd8b4aff37971c090c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:31,250 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:31,250 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/B, priority=13, startTime=1731213511214; duration=0sec 2024-11-10T04:38:31,250 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:31,250 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:B 2024-11-10T04:38:31,251 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:31,251 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:31,251 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/C is initiating minor compaction (all files) 2024-11-10T04:38:31,251 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/C in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,252 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/fd286706d0a54349af0206169dcf1e35, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8ce8ad6a221b4a5e82ec5fe50ec94de3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/07b8c5d32a6e407980e62471dbdc5fa2] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=33.7 K 2024-11-10T04:38:31,252 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fd286706d0a54349af0206169dcf1e35, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731213505189 2024-11-10T04:38:31,252 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce8ad6a221b4a5e82ec5fe50ec94de3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731213505887 2024-11-10T04:38:31,253 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b8c5d32a6e407980e62471dbdc5fa2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213508099 2024-11-10T04:38:31,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742214_1390 (size=4469) 2024-11-10T04:38:31,264 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:31,265 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/ea7ab00a64774545af3351c7031eab87 is 50, key is test_row_0/C:col10/1731213508099/Put/seqid=0 2024-11-10T04:38:31,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742215_1391 (size=12731) 2024-11-10T04:38:31,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T04:38:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,349 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T04:38:31,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110504b3a18c05440e399b3220f0340b946_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213510406/Put/seqid=0 2024-11-10T04:38:31,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742216_1392 (size=12454) 2024-11-10T04:38:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T04:38:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. as already flushing 2024-11-10T04:38:31,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213571611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213571612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213571613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213571614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,659 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:31,659 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/088c5381f9734fa4921f3449fe98daff is 175, key is test_row_0/A:col10/1731213508099/Put/seqid=0 2024-11-10T04:38:31,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742217_1393 (size=31792) 2024-11-10T04:38:31,676 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/ea7ab00a64774545af3351c7031eab87 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/ea7ab00a64774545af3351c7031eab87 2024-11-10T04:38:31,680 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/C of 6d816208b10c05c54c05576e9800ef1c into ea7ab00a64774545af3351c7031eab87(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:31,680 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:31,680 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/C, priority=13, startTime=1731213511214; duration=0sec 2024-11-10T04:38:31,680 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:31,680 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:C 2024-11-10T04:38:31,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33814 deadline: 1731213571731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33772 deadline: 1731213571731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33756 deadline: 1731213571731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33766 deadline: 1731213571731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:38:31,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:31,765 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110504b3a18c05440e399b3220f0340b946_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110504b3a18c05440e399b3220f0340b946_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/b1cd58a8676f447da4ff389046de3d39, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/b1cd58a8676f447da4ff389046de3d39 is 175, key is test_row_0/A:col10/1731213510406/Put/seqid=0 2024-11-10T04:38:31,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742218_1394 (size=31255) 2024-11-10T04:38:31,770 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/b1cd58a8676f447da4ff389046de3d39 2024-11-10T04:38:31,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/cb8a9d9b5abc49f1a27b69d7fa56fbdf is 50, key is test_row_0/B:col10/1731213510406/Put/seqid=0 2024-11-10T04:38:31,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742219_1395 (size=12301) 2024-11-10T04:38:31,784 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/cb8a9d9b5abc49f1a27b69d7fa56fbdf 2024-11-10T04:38:31,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/2cb365f8d5b84899bca30bb31e876b6f is 50, key is test_row_0/C:col10/1731213510406/Put/seqid=0 2024-11-10T04:38:31,793 INFO [Thread-1486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1486. 2024-11-10T04:38:31,793 DEBUG [Thread-1486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:31,793 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742220_1396 (size=12301) 2024-11-10T04:38:31,793 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:31,794 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/2cb365f8d5b84899bca30bb31e876b6f 2024-11-10T04:38:31,796 INFO [Thread-1480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1480. 2024-11-10T04:38:31,796 INFO [Thread-1482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1482. 2024-11-10T04:38:31,796 DEBUG [Thread-1480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:31,796 DEBUG [Thread-1482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:31,796 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,796 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,796 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,796 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:31,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:31,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/b1cd58a8676f447da4ff389046de3d39 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/b1cd58a8676f447da4ff389046de3d39 2024-11-10T04:38:31,803 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/b1cd58a8676f447da4ff389046de3d39, entries=150, sequenceid=274, filesize=30.5 K 2024-11-10T04:38:31,804 INFO [Thread-1488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1488. 2024-11-10T04:38:31,804 DEBUG [Thread-1488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:31,804 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:31,804 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/cb8a9d9b5abc49f1a27b69d7fa56fbdf as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cb8a9d9b5abc49f1a27b69d7fa56fbdf 2024-11-10T04:38:31,808 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cb8a9d9b5abc49f1a27b69d7fa56fbdf, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T04:38:31,809 INFO [Thread-1484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1484. 2024-11-10T04:38:31,809 DEBUG [Thread-1484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:31,809 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,809 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:31,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/2cb365f8d5b84899bca30bb31e876b6f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/2cb365f8d5b84899bca30bb31e876b6f 2024-11-10T04:38:31,813 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/2cb365f8d5b84899bca30bb31e876b6f, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T04:38:31,814 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6d816208b10c05c54c05576e9800ef1c in 465ms, sequenceid=274, compaction requested=false 2024-11-10T04:38:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-10T04:38:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-10T04:38:31,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-10T04:38:31,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 928 msec 2024-11-10T04:38:31,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 936 msec 2024-11-10T04:38:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:31,943 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:31,943 INFO [Thread-1477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1477. 2024-11-10T04:38:31,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T04:38:31,943 DEBUG [Thread-1477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:31,943 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,943 INFO [Thread-1471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1471. 2024-11-10T04:38:31,943 INFO [Thread-1469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1469. 2024-11-10T04:38:31,943 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,943 DEBUG [Thread-1471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:31,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:31,943 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,943 DEBUG [Thread-1469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:31,943 DEBUG [Thread-1469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,943 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,943 DEBUG [Thread-1469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:31,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:31,944 INFO [Thread-1473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1473. 2024-11-10T04:38:31,944 DEBUG [Thread-1473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:31,944 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,944 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:31,944 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:31,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f74fb017e5a14b338d0c01c79c24fd06_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_0/A:col10/1731213511941/Put/seqid=0 2024-11-10T04:38:31,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742221_1397 (size=12454) 2024-11-10T04:38:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T04:38:32,021 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:32,067 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/088c5381f9734fa4921f3449fe98daff as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/088c5381f9734fa4921f3449fe98daff 2024-11-10T04:38:32,071 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/A of 6d816208b10c05c54c05576e9800ef1c into 088c5381f9734fa4921f3449fe98daff(size=31.0 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:32,071 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:32,071 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/A, priority=13, startTime=1731213511214; duration=0sec 2024-11-10T04:38:32,071 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:32,071 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:A 2024-11-10T04:38:32,355 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:32,358 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f74fb017e5a14b338d0c01c79c24fd06_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f74fb017e5a14b338d0c01c79c24fd06_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:32,359 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/702b9093892d4996bb71cd55bd0463b8, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:32,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/702b9093892d4996bb71cd55bd0463b8 is 175, key is test_row_0/A:col10/1731213511941/Put/seqid=0 2024-11-10T04:38:32,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742222_1398 (size=31255) 2024-11-10T04:38:32,763 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/702b9093892d4996bb71cd55bd0463b8 2024-11-10T04:38:32,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/11b80a0d471c451d9031676ed8c7232c is 50, key is test_row_0/B:col10/1731213511941/Put/seqid=0 2024-11-10T04:38:32,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742223_1399 (size=12301) 2024-11-10T04:38:33,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/11b80a0d471c451d9031676ed8c7232c 2024-11-10T04:38:33,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/159dd9d1099447738cfd2fa9de5ae2a7 is 50, key is test_row_0/C:col10/1731213511941/Put/seqid=0 2024-11-10T04:38:33,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742224_1400 (size=12301) 2024-11-10T04:38:33,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/159dd9d1099447738cfd2fa9de5ae2a7 2024-11-10T04:38:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/702b9093892d4996bb71cd55bd0463b8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/702b9093892d4996bb71cd55bd0463b8 2024-11-10T04:38:33,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/702b9093892d4996bb71cd55bd0463b8, entries=150, sequenceid=291, filesize=30.5 K 2024-11-10T04:38:33,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/11b80a0d471c451d9031676ed8c7232c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/11b80a0d471c451d9031676ed8c7232c 2024-11-10T04:38:33,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/11b80a0d471c451d9031676ed8c7232c, entries=150, sequenceid=291, filesize=12.0 K 2024-11-10T04:38:33,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/159dd9d1099447738cfd2fa9de5ae2a7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/159dd9d1099447738cfd2fa9de5ae2a7 2024-11-10T04:38:33,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/159dd9d1099447738cfd2fa9de5ae2a7, entries=150, sequenceid=291, filesize=12.0 K 2024-11-10T04:38:33,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for 6d816208b10c05c54c05576e9800ef1c in 1653ms, sequenceid=291, compaction requested=true 2024-11-10T04:38:33,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:33,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:33,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:33,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:33,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:33,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d816208b10c05c54c05576e9800ef1c:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:33,596 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:33,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:33,596 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:33,597 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94302 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:33,597 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:33,597 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/A is initiating minor compaction (all files) 2024-11-10T04:38:33,597 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/B is initiating minor compaction (all files) 2024-11-10T04:38:33,597 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/A in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:33,597 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/B in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:33,597 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c51feb96b06340dd8b4aff37971c090c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cb8a9d9b5abc49f1a27b69d7fa56fbdf, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/11b80a0d471c451d9031676ed8c7232c] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=36.5 K 2024-11-10T04:38:33,597 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/088c5381f9734fa4921f3449fe98daff, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/b1cd58a8676f447da4ff389046de3d39, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/702b9093892d4996bb71cd55bd0463b8] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=92.1 K 2024-11-10T04:38:33,597 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:33,597 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/088c5381f9734fa4921f3449fe98daff, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/b1cd58a8676f447da4ff389046de3d39, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/702b9093892d4996bb71cd55bd0463b8] 2024-11-10T04:38:33,597 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c51feb96b06340dd8b4aff37971c090c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213505887 2024-11-10T04:38:33,597 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 088c5381f9734fa4921f3449fe98daff, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213505887 2024-11-10T04:38:33,598 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cb8a9d9b5abc49f1a27b69d7fa56fbdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731213510365 2024-11-10T04:38:33,598 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1cd58a8676f447da4ff389046de3d39, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731213510365 2024-11-10T04:38:33,598 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b80a0d471c451d9031676ed8c7232c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731213511941 2024-11-10T04:38:33,598 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702b9093892d4996bb71cd55bd0463b8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731213511941 2024-11-10T04:38:33,604 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:33,605 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:33,605 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411101a0d1f5e6b4f4b31a55d9df14b762784_6d816208b10c05c54c05576e9800ef1c store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:33,605 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/a03e662863594fec826ced457d4d4910 is 50, key is test_row_0/B:col10/1731213511941/Put/seqid=0 2024-11-10T04:38:33,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742225_1401 (size=12983) 2024-11-10T04:38:33,610 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411101a0d1f5e6b4f4b31a55d9df14b762784_6d816208b10c05c54c05576e9800ef1c, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:33,610 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101a0d1f5e6b4f4b31a55d9df14b762784_6d816208b10c05c54c05576e9800ef1c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:33,615 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/a03e662863594fec826ced457d4d4910 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/a03e662863594fec826ced457d4d4910 2024-11-10T04:38:33,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742226_1402 (size=4469) 2024-11-10T04:38:33,618 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#A#compaction#337 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:33,618 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/c4ceddcd0e0b410ba04a33fd95a133e7 is 175, key is test_row_0/A:col10/1731213511941/Put/seqid=0 2024-11-10T04:38:33,619 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/B of 6d816208b10c05c54c05576e9800ef1c into a03e662863594fec826ced457d4d4910(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:33,619 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:33,619 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/B, priority=13, startTime=1731213513596; duration=0sec 2024-11-10T04:38:33,619 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:33,619 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:B 2024-11-10T04:38:33,619 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:33,620 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:33,620 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 6d816208b10c05c54c05576e9800ef1c/C is initiating minor compaction (all files) 2024-11-10T04:38:33,620 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d816208b10c05c54c05576e9800ef1c/C in TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:33,620 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/ea7ab00a64774545af3351c7031eab87, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/2cb365f8d5b84899bca30bb31e876b6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/159dd9d1099447738cfd2fa9de5ae2a7] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp, totalSize=36.5 K 2024-11-10T04:38:33,621 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ea7ab00a64774545af3351c7031eab87, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731213505887 2024-11-10T04:38:33,621 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb365f8d5b84899bca30bb31e876b6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731213510365 2024-11-10T04:38:33,621 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 159dd9d1099447738cfd2fa9de5ae2a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731213511941 2024-11-10T04:38:33,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742227_1403 (size=31937) 2024-11-10T04:38:33,628 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/c4ceddcd0e0b410ba04a33fd95a133e7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/c4ceddcd0e0b410ba04a33fd95a133e7 2024-11-10T04:38:33,635 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/A of 6d816208b10c05c54c05576e9800ef1c into c4ceddcd0e0b410ba04a33fd95a133e7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:33,635 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:33,636 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/A, priority=13, startTime=1731213513596; duration=0sec 2024-11-10T04:38:33,636 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:33,636 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:A 2024-11-10T04:38:33,636 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d816208b10c05c54c05576e9800ef1c#C#compaction#338 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:33,637 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/762e160e562c4c9881ee8b65ba3fc9d7 is 50, key is test_row_0/C:col10/1731213511941/Put/seqid=0 2024-11-10T04:38:33,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742228_1404 (size=12983) 2024-11-10T04:38:33,739 WARN [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-10T04:38:33,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a24d8373ae164430b2d2667ab9cd68f7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/9d2c010923d14d2386bea7506267b6c7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/718989cd125944c19585a90a5d44911f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a120995283914269999be978581e9cf3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/88d433a3de244f9b858e2d34815bf7d4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/7f41e231ca68428b9de2483320953139, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/c1365188d81242c599be0823428f1e37, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2493042381b4d35984e13b0ffc5052e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2c2a51562a64ff79815127f70d852d5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/47597b20710f4f4dba14de7583d8f1d6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/78831fd46f304b448de38737aeddef8a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/ba20f68cc848489aabd1952d19281e02, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f6c8030ffdda41c5aa410b4ca22c648d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/d865bd5441ca49ecb22fe78dd71337e7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/40666f2d5405423094b72c55f0b923e1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/64b6f30e746a447bacbbc0dbe739136f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/8135f9ae13b245e5b977ce65304cdfe8, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/088c5381f9734fa4921f3449fe98daff, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/b1cd58a8676f447da4ff389046de3d39, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/702b9093892d4996bb71cd55bd0463b8] to archive 2024-11-10T04:38:33,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:38:33,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a24d8373ae164430b2d2667ab9cd68f7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a24d8373ae164430b2d2667ab9cd68f7 2024-11-10T04:38:33,744 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/9d2c010923d14d2386bea7506267b6c7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/9d2c010923d14d2386bea7506267b6c7 2024-11-10T04:38:33,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/718989cd125944c19585a90a5d44911f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/718989cd125944c19585a90a5d44911f 2024-11-10T04:38:33,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a120995283914269999be978581e9cf3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a120995283914269999be978581e9cf3 2024-11-10T04:38:33,746 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/88d433a3de244f9b858e2d34815bf7d4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/88d433a3de244f9b858e2d34815bf7d4 2024-11-10T04:38:33,747 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/7f41e231ca68428b9de2483320953139 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/7f41e231ca68428b9de2483320953139 2024-11-10T04:38:33,748 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/c1365188d81242c599be0823428f1e37 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/c1365188d81242c599be0823428f1e37 2024-11-10T04:38:33,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2493042381b4d35984e13b0ffc5052e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2493042381b4d35984e13b0ffc5052e 2024-11-10T04:38:33,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2c2a51562a64ff79815127f70d852d5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f2c2a51562a64ff79815127f70d852d5 2024-11-10T04:38:33,750 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/47597b20710f4f4dba14de7583d8f1d6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/47597b20710f4f4dba14de7583d8f1d6 2024-11-10T04:38:33,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/78831fd46f304b448de38737aeddef8a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/78831fd46f304b448de38737aeddef8a 2024-11-10T04:38:33,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/ba20f68cc848489aabd1952d19281e02 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/ba20f68cc848489aabd1952d19281e02 2024-11-10T04:38:33,753 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f6c8030ffdda41c5aa410b4ca22c648d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/f6c8030ffdda41c5aa410b4ca22c648d 2024-11-10T04:38:33,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/d865bd5441ca49ecb22fe78dd71337e7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/d865bd5441ca49ecb22fe78dd71337e7 2024-11-10T04:38:33,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/40666f2d5405423094b72c55f0b923e1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/40666f2d5405423094b72c55f0b923e1 2024-11-10T04:38:33,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/64b6f30e746a447bacbbc0dbe739136f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/64b6f30e746a447bacbbc0dbe739136f 2024-11-10T04:38:33,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/8135f9ae13b245e5b977ce65304cdfe8 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/8135f9ae13b245e5b977ce65304cdfe8 2024-11-10T04:38:33,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/088c5381f9734fa4921f3449fe98daff to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/088c5381f9734fa4921f3449fe98daff 2024-11-10T04:38:33,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/b1cd58a8676f447da4ff389046de3d39 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/b1cd58a8676f447da4ff389046de3d39 2024-11-10T04:38:33,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/702b9093892d4996bb71cd55bd0463b8 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/702b9093892d4996bb71cd55bd0463b8 2024-11-10T04:38:33,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/fb767dc07286490fa26b31a4e68d5305, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/f826d39305b64b4db542ddff0255cf2d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/41c6e31c7344424db781b55af9545287, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cae189d5fd6846a29789d8ebbc14e70e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/e8706906ff714b649658bc973153d571, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/916bc9523b914bd99caeb94bcc6d868b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/a144a436056343c3920023a8b2a85a75, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/1b6528323eb34a979ed3a2d8008799a7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/dcfa5bd133eb498782fa960a798d78b3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/97c00acc49514b4ea83621c06f9a835a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/16e598a5179a42b087f37bd6d38140f1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c0c5327782ac43e7bdf862c7a0b06c44, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/2e841daf9295457aa553777225d8e629, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/0710778ea65e40cfb284d28798d5f64d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/143c5a8a636c47d3b5a1cb3da52e7cba, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/174948c930394518a58f2d732f0c7e43, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c51feb96b06340dd8b4aff37971c090c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/779eca82e85b44059a806c886542aaf3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cb8a9d9b5abc49f1a27b69d7fa56fbdf, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/11b80a0d471c451d9031676ed8c7232c] to archive 2024-11-10T04:38:33,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:38:33,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/fb767dc07286490fa26b31a4e68d5305 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/fb767dc07286490fa26b31a4e68d5305 2024-11-10T04:38:33,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/f826d39305b64b4db542ddff0255cf2d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/f826d39305b64b4db542ddff0255cf2d 2024-11-10T04:38:33,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/41c6e31c7344424db781b55af9545287 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/41c6e31c7344424db781b55af9545287 2024-11-10T04:38:33,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cae189d5fd6846a29789d8ebbc14e70e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cae189d5fd6846a29789d8ebbc14e70e 2024-11-10T04:38:33,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/e8706906ff714b649658bc973153d571 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/e8706906ff714b649658bc973153d571 2024-11-10T04:38:33,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/916bc9523b914bd99caeb94bcc6d868b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/916bc9523b914bd99caeb94bcc6d868b 2024-11-10T04:38:33,767 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/a144a436056343c3920023a8b2a85a75 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/a144a436056343c3920023a8b2a85a75 2024-11-10T04:38:33,768 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/1b6528323eb34a979ed3a2d8008799a7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/1b6528323eb34a979ed3a2d8008799a7 2024-11-10T04:38:33,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/dcfa5bd133eb498782fa960a798d78b3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/dcfa5bd133eb498782fa960a798d78b3 2024-11-10T04:38:33,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/97c00acc49514b4ea83621c06f9a835a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/97c00acc49514b4ea83621c06f9a835a 2024-11-10T04:38:33,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/16e598a5179a42b087f37bd6d38140f1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/16e598a5179a42b087f37bd6d38140f1 2024-11-10T04:38:33,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c0c5327782ac43e7bdf862c7a0b06c44 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c0c5327782ac43e7bdf862c7a0b06c44 2024-11-10T04:38:33,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/2e841daf9295457aa553777225d8e629 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/2e841daf9295457aa553777225d8e629 2024-11-10T04:38:33,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/0710778ea65e40cfb284d28798d5f64d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/0710778ea65e40cfb284d28798d5f64d 2024-11-10T04:38:33,773 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/143c5a8a636c47d3b5a1cb3da52e7cba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/143c5a8a636c47d3b5a1cb3da52e7cba 2024-11-10T04:38:33,774 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/174948c930394518a58f2d732f0c7e43 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/174948c930394518a58f2d732f0c7e43 2024-11-10T04:38:33,775 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c51feb96b06340dd8b4aff37971c090c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/c51feb96b06340dd8b4aff37971c090c 2024-11-10T04:38:33,775 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/779eca82e85b44059a806c886542aaf3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/779eca82e85b44059a806c886542aaf3 2024-11-10T04:38:33,776 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cb8a9d9b5abc49f1a27b69d7fa56fbdf to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/cb8a9d9b5abc49f1a27b69d7fa56fbdf 2024-11-10T04:38:33,777 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/11b80a0d471c451d9031676ed8c7232c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/11b80a0d471c451d9031676ed8c7232c 2024-11-10T04:38:33,779 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/b01d0b570b2b40aa93b98ddfce9c135a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/0487099194db48ed9ec5417658ba69eb, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/82aa5c3859d248d7a2848646b3ecc889, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/252afd3c1a184d6cba4fb9d052dcf6c9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a05e589c53c940fd8ca4f841990d5892, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/6a57247ac01f4b948fee97365852051e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8157a8ee178047318bbcdb0911ecffb5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/f2f5f13c36264086ab5b90a1cc27107d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/cd2ae10d62c74254844748663f951336, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/78e3b55035aa44fcb596985c0256b997, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a9fb47a536a14d4dbb3d47633b4e7ea9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/572c48ba49194c6291bf6fdcc9f591f0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/9cb6a905970341e3afa9b49606249695, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/fd286706d0a54349af0206169dcf1e35, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/1f38e14cbb914451b0b0a510f97bb370, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8ce8ad6a221b4a5e82ec5fe50ec94de3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/07b8c5d32a6e407980e62471dbdc5fa2] to archive 2024-11-10T04:38:33,779 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:38:33,780 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/b01d0b570b2b40aa93b98ddfce9c135a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/b01d0b570b2b40aa93b98ddfce9c135a 2024-11-10T04:38:33,781 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/0487099194db48ed9ec5417658ba69eb to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/0487099194db48ed9ec5417658ba69eb 2024-11-10T04:38:33,782 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/82aa5c3859d248d7a2848646b3ecc889 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/82aa5c3859d248d7a2848646b3ecc889 2024-11-10T04:38:33,783 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/252afd3c1a184d6cba4fb9d052dcf6c9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/252afd3c1a184d6cba4fb9d052dcf6c9 2024-11-10T04:38:33,783 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a05e589c53c940fd8ca4f841990d5892 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a05e589c53c940fd8ca4f841990d5892 2024-11-10T04:38:33,784 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/6a57247ac01f4b948fee97365852051e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/6a57247ac01f4b948fee97365852051e 2024-11-10T04:38:33,785 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8157a8ee178047318bbcdb0911ecffb5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8157a8ee178047318bbcdb0911ecffb5 2024-11-10T04:38:33,786 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/f2f5f13c36264086ab5b90a1cc27107d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/f2f5f13c36264086ab5b90a1cc27107d 2024-11-10T04:38:33,786 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/cd2ae10d62c74254844748663f951336 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/cd2ae10d62c74254844748663f951336 2024-11-10T04:38:33,787 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/78e3b55035aa44fcb596985c0256b997 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/78e3b55035aa44fcb596985c0256b997 2024-11-10T04:38:33,788 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a9fb47a536a14d4dbb3d47633b4e7ea9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/a9fb47a536a14d4dbb3d47633b4e7ea9 2024-11-10T04:38:33,789 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/572c48ba49194c6291bf6fdcc9f591f0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/572c48ba49194c6291bf6fdcc9f591f0 2024-11-10T04:38:33,789 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/9cb6a905970341e3afa9b49606249695 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/9cb6a905970341e3afa9b49606249695 2024-11-10T04:38:33,790 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/fd286706d0a54349af0206169dcf1e35 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/fd286706d0a54349af0206169dcf1e35 2024-11-10T04:38:33,791 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/1f38e14cbb914451b0b0a510f97bb370 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/1f38e14cbb914451b0b0a510f97bb370 2024-11-10T04:38:33,792 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8ce8ad6a221b4a5e82ec5fe50ec94de3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/8ce8ad6a221b4a5e82ec5fe50ec94de3 2024-11-10T04:38:33,792 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/d1d9e3766cb1:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/07b8c5d32a6e407980e62471dbdc5fa2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/07b8c5d32a6e407980e62471dbdc5fa2 2024-11-10T04:38:34,045 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/762e160e562c4c9881ee8b65ba3fc9d7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/762e160e562c4c9881ee8b65ba3fc9d7 2024-11-10T04:38:34,049 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d816208b10c05c54c05576e9800ef1c/C of 6d816208b10c05c54c05576e9800ef1c into 762e160e562c4c9881ee8b65ba3fc9d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:34,049 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d816208b10c05c54c05576e9800ef1c: 2024-11-10T04:38:34,049 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., storeName=6d816208b10c05c54c05576e9800ef1c/C, priority=13, startTime=1731213513596; duration=0sec 2024-11-10T04:38:34,049 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:34,049 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d816208b10c05c54c05576e9800ef1c:C 2024-11-10T04:38:38,893 INFO [Thread-1475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1475. 2024-11-10T04:38:38,893 DEBUG [Thread-1475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:38:38,893 DEBUG [Thread-1475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:38,893 DEBUG [Thread-1475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:38,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:38,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T04:38:38,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-10T04:38:38,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-10T04:38:38,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-10T04:38:38,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-10T04:38:38,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-10T04:38:38,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T04:38:38,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T04:38:38,893 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1130 2024-11-10T04:38:38,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3388 rows 2024-11-10T04:38:38,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1125 2024-11-10T04:38:38,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3371 rows 2024-11-10T04:38:38,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1129 2024-11-10T04:38:38,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3385 rows 2024-11-10T04:38:38,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1126 2024-11-10T04:38:38,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3376 rows 2024-11-10T04:38:38,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1130 2024-11-10T04:38:38,894 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3384 rows 2024-11-10T04:38:38,894 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:38:38,894 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:38:38,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:38,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:38,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:38:38,895 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T04:38:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T04:38:38,898 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213518898"}]},"ts":"1731213518898"} 2024-11-10T04:38:38,899 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T04:38:38,899 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T04:38:38,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T04:38:38,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, UNASSIGN}] 2024-11-10T04:38:38,901 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, UNASSIGN 2024-11-10T04:38:38,902 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=6d816208b10c05c54c05576e9800ef1c, regionState=CLOSING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:38,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, UNASSIGN because future has completed 2024-11-10T04:38:38,903 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T04:38:38,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:38:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T04:38:39,055 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:39,055 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T04:38:39,055 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 6d816208b10c05c54c05576e9800ef1c, disabling compactions & flushes 2024-11-10T04:38:39,055 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:39,055 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:39,055 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. after waiting 0 ms 2024-11-10T04:38:39,055 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:39,055 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 6d816208b10c05c54c05576e9800ef1c 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-10T04:38:39,060 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110517ab2850df04213838d3b9032d244e2_6d816208b10c05c54c05576e9800ef1c is 50, key is test_row_2/A:col10/1731213518891/Put/seqid=0 2024-11-10T04:38:39,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742229_1405 (size=7374) 2024-11-10T04:38:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T04:38:39,464 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:39,467 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110517ab2850df04213838d3b9032d244e2_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110517ab2850df04213838d3b9032d244e2_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:39,468 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a43b572536e9457ab8c8c33dbfef663d, store: [table=TestAcidGuarantees family=A region=6d816208b10c05c54c05576e9800ef1c] 2024-11-10T04:38:39,468 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a43b572536e9457ab8c8c33dbfef663d is 175, key is test_row_2/A:col10/1731213518891/Put/seqid=0 2024-11-10T04:38:39,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742230_1406 (size=13865) 2024-11-10T04:38:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T04:38:39,872 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a43b572536e9457ab8c8c33dbfef663d 2024-11-10T04:38:39,878 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/58ca5c8469484d8dae84c400138aa0c2 is 50, key is test_row_2/B:col10/1731213518891/Put/seqid=0 2024-11-10T04:38:39,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742231_1407 (size=7415) 2024-11-10T04:38:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T04:38:40,281 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/58ca5c8469484d8dae84c400138aa0c2 2024-11-10T04:38:40,287 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/820daba3adc945afa7c0daaef991b2ca is 50, key is test_row_2/C:col10/1731213518891/Put/seqid=0 2024-11-10T04:38:40,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742232_1408 (size=7415) 2024-11-10T04:38:40,690 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/820daba3adc945afa7c0daaef991b2ca 2024-11-10T04:38:40,693 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/A/a43b572536e9457ab8c8c33dbfef663d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a43b572536e9457ab8c8c33dbfef663d 2024-11-10T04:38:40,696 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a43b572536e9457ab8c8c33dbfef663d, entries=50, sequenceid=299, filesize=13.5 K 2024-11-10T04:38:40,697 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/B/58ca5c8469484d8dae84c400138aa0c2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/58ca5c8469484d8dae84c400138aa0c2 2024-11-10T04:38:40,699 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/58ca5c8469484d8dae84c400138aa0c2, entries=50, sequenceid=299, filesize=7.2 K 2024-11-10T04:38:40,700 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/.tmp/C/820daba3adc945afa7c0daaef991b2ca as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/820daba3adc945afa7c0daaef991b2ca 2024-11-10T04:38:40,702 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/820daba3adc945afa7c0daaef991b2ca, entries=50, sequenceid=299, filesize=7.2 K 2024-11-10T04:38:40,703 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 6d816208b10c05c54c05576e9800ef1c in 1648ms, sequenceid=299, compaction requested=false 2024-11-10T04:38:40,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/ea7ab00a64774545af3351c7031eab87, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/2cb365f8d5b84899bca30bb31e876b6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/159dd9d1099447738cfd2fa9de5ae2a7] to archive 2024-11-10T04:38:40,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:38:40,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/ea7ab00a64774545af3351c7031eab87 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/ea7ab00a64774545af3351c7031eab87 2024-11-10T04:38:40,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/2cb365f8d5b84899bca30bb31e876b6f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/2cb365f8d5b84899bca30bb31e876b6f 2024-11-10T04:38:40,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/159dd9d1099447738cfd2fa9de5ae2a7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/159dd9d1099447738cfd2fa9de5ae2a7 2024-11-10T04:38:40,711 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/recovered.edits/302.seqid, newMaxSeqId=302, maxSeqId=4 2024-11-10T04:38:40,712 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. 2024-11-10T04:38:40,712 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 6d816208b10c05c54c05576e9800ef1c: Waiting for close lock at 1731213519055Running coprocessor pre-close hooks at 1731213519055Disabling compacts and flushes for region at 1731213519055Disabling writes for close at 1731213519055Obtaining lock to block concurrent updates at 1731213519055Preparing flush snapshotting stores in 6d816208b10c05c54c05576e9800ef1c at 1731213519055Finished memstore snapshotting TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1731213519055Flushing stores of TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c. at 1731213519056 (+1 ms)Flushing 6d816208b10c05c54c05576e9800ef1c/A: creating writer at 1731213519056Flushing mob file 6d816208b10c05c54c05576e9800ef1c/A: appending metadata at 1731213519060 (+4 ms)Flushing mob file 6d816208b10c05c54c05576e9800ef1c/A: closing flushed file at 1731213519060Flushing 6d816208b10c05c54c05576e9800ef1c/A: appending metadata at 1731213519468 (+408 ms)Flushing 6d816208b10c05c54c05576e9800ef1c/A: closing flushed file at 1731213519468Flushing 6d816208b10c05c54c05576e9800ef1c/B: creating writer at 1731213519875 (+407 ms)Flushing 6d816208b10c05c54c05576e9800ef1c/B: appending metadata at 1731213519877 (+2 ms)Flushing 6d816208b10c05c54c05576e9800ef1c/B: closing flushed file at 1731213519877Flushing 6d816208b10c05c54c05576e9800ef1c/C: creating writer at 1731213520284 (+407 ms)Flushing 6d816208b10c05c54c05576e9800ef1c/C: appending metadata at 1731213520286 (+2 ms)Flushing 6d816208b10c05c54c05576e9800ef1c/C: closing flushed file at 1731213520286Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1133091b: reopening flushed file at 1731213520693 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18c753eb: reopening flushed file at 1731213520696 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6670bcfb: reopening flushed file at 1731213520699 (+3 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 6d816208b10c05c54c05576e9800ef1c in 1648ms, sequenceid=299, compaction requested=false at 1731213520703 (+4 ms)Writing region close event to WAL at 1731213520708 (+5 ms)Running coprocessor post-close hooks at 1731213520712 (+4 ms)Closed at 1731213520712 2024-11-10T04:38:40,713 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:40,714 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=6d816208b10c05c54c05576e9800ef1c, regionState=CLOSED 2024-11-10T04:38:40,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:38:40,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-10T04:38:40,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 6d816208b10c05c54c05576e9800ef1c, server=d1d9e3766cb1,45693,1731213391842 in 1.8130 sec 2024-11-10T04:38:40,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-10T04:38:40,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d816208b10c05c54c05576e9800ef1c, UNASSIGN in 1.8170 sec 2024-11-10T04:38:40,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T04:38:40,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8200 sec 2024-11-10T04:38:40,722 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213520722"}]},"ts":"1731213520722"} 2024-11-10T04:38:40,723 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T04:38:40,723 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T04:38:40,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8290 sec 2024-11-10T04:38:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T04:38:41,041 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:41,042 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T04:38:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:41,044 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T04:38:41,045 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:41,047 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,049 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/recovered.edits] 2024-11-10T04:38:41,051 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a43b572536e9457ab8c8c33dbfef663d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/a43b572536e9457ab8c8c33dbfef663d 2024-11-10T04:38:41,052 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/c4ceddcd0e0b410ba04a33fd95a133e7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/A/c4ceddcd0e0b410ba04a33fd95a133e7 2024-11-10T04:38:41,053 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/58ca5c8469484d8dae84c400138aa0c2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/58ca5c8469484d8dae84c400138aa0c2 2024-11-10T04:38:41,054 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/a03e662863594fec826ced457d4d4910 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/B/a03e662863594fec826ced457d4d4910 2024-11-10T04:38:41,056 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/762e160e562c4c9881ee8b65ba3fc9d7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/762e160e562c4c9881ee8b65ba3fc9d7 2024-11-10T04:38:41,057 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/820daba3adc945afa7c0daaef991b2ca to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/C/820daba3adc945afa7c0daaef991b2ca 2024-11-10T04:38:41,059 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/recovered.edits/302.seqid to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c/recovered.edits/302.seqid 2024-11-10T04:38:41,059 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,059 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T04:38:41,060 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T04:38:41,060 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T04:38:41,062 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111019f9be000874448680c4ebc51ee8c917_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111019f9be000874448680c4ebc51ee8c917_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,063 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101db0de8e87654f4b8abea659ba2a8980_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101db0de8e87654f4b8abea659ba2a8980_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,064 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102c0f895a5b014e55a61a094d778a87b9_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102c0f895a5b014e55a61a094d778a87b9_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,065 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102da0b8a5146148b890808c702dfc2108_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102da0b8a5146148b890808c702dfc2108_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,066 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103a2d9bbd0c9543a3b0c1d30082b9b56d_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103a2d9bbd0c9543a3b0c1d30082b9b56d_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,067 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110490482d8e9df4c6e9d2a99eb76568140_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110490482d8e9df4c6e9d2a99eb76568140_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,068 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110504b3a18c05440e399b3220f0340b946_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110504b3a18c05440e399b3220f0340b946_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,069 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110517ab2850df04213838d3b9032d244e2_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110517ab2850df04213838d3b9032d244e2_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,069 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108cd3c8ab2ff245dcbc1eb16993ebb4c8_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108cd3c8ab2ff245dcbc1eb16993ebb4c8_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,070 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109a6d014541fb45e58c0088c6a278d800_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109a6d014541fb45e58c0088c6a278d800_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,071 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109c7ba6ea8d5b4709a15a51618da94c85_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109c7ba6ea8d5b4709a15a51618da94c85_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,072 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109cc6b9f9533249a7b671fba0ccb6d9f2_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109cc6b9f9533249a7b671fba0ccb6d9f2_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,073 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b77ba567bb494c4983d2ae4234344c66_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b77ba567bb494c4983d2ae4234344c66_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,074 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dceb50111125490c84da2f5478005da5_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dceb50111125490c84da2f5478005da5_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,075 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e19cde29157b4b43b8bf7653f94c49a2_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e19cde29157b4b43b8bf7653f94c49a2_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,075 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f74fb017e5a14b338d0c01c79c24fd06_6d816208b10c05c54c05576e9800ef1c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f74fb017e5a14b338d0c01c79c24fd06_6d816208b10c05c54c05576e9800ef1c 2024-11-10T04:38:41,076 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T04:38:41,078 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:41,079 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T04:38:41,081 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T04:38:41,081 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:41,081 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T04:38:41,082 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731213521081"}]},"ts":"9223372036854775807"} 2024-11-10T04:38:41,083 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T04:38:41,083 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6d816208b10c05c54c05576e9800ef1c, NAME => 'TestAcidGuarantees,,1731213488670.6d816208b10c05c54c05576e9800ef1c.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T04:38:41,083 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T04:38:41,083 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731213521083"}]},"ts":"9223372036854775807"} 2024-11-10T04:38:41,084 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T04:38:41,085 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:41,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-10T04:38:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T04:38:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T04:38:41,151 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:41,160 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 239), OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=275 (was 258) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6349 (was 6401) 2024-11-10T04:38:41,168 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=275, ProcessCount=11, AvailableMemoryMB=6349 2024-11-10T04:38:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T04:38:41,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:38:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T04:38:41,171 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:38:41,171 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:41,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-10T04:38:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-10T04:38:41,171 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:38:41,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742233_1409 (size=959) 2024-11-10T04:38:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-10T04:38:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-10T04:38:41,578 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8 2024-11-10T04:38:41,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742234_1410 (size=53) 2024-11-10T04:38:41,584 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T04:38:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-10T04:38:41,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:38:41,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 12628841f5cc772abac56da27a91cfca, disabling compactions & flushes 2024-11-10T04:38:41,983 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:41,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:41,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. after waiting 0 ms 2024-11-10T04:38:41,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:41,983 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:41,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 12628841f5cc772abac56da27a91cfca: Waiting for close lock at 1731213521983Disabling compacts and flushes for region at 1731213521983Disabling writes for close at 1731213521983Writing region close event to WAL at 1731213521983Closed at 1731213521983 2024-11-10T04:38:41,984 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:38:41,984 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731213521984"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731213521984"}]},"ts":"1731213521984"} 2024-11-10T04:38:41,985 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:38:41,986 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:38:41,986 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213521986"}]},"ts":"1731213521986"} 2024-11-10T04:38:41,987 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T04:38:41,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12628841f5cc772abac56da27a91cfca, ASSIGN}] 2024-11-10T04:38:41,988 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12628841f5cc772abac56da27a91cfca, ASSIGN 2024-11-10T04:38:41,989 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12628841f5cc772abac56da27a91cfca, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,45693,1731213391842; forceNewPlan=false, retain=false 2024-11-10T04:38:42,139 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=12628841f5cc772abac56da27a91cfca, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:42,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12628841f5cc772abac56da27a91cfca, ASSIGN because future has completed 2024-11-10T04:38:42,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:38:42,295 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:42,295 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:38:42,296 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,296 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:38:42,296 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,296 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,297 INFO [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,298 INFO [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12628841f5cc772abac56da27a91cfca columnFamilyName A 2024-11-10T04:38:42,298 DEBUG [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:42,298 INFO [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] regionserver.HStore(327): Store=12628841f5cc772abac56da27a91cfca/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:38:42,299 INFO [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,299 INFO [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12628841f5cc772abac56da27a91cfca columnFamilyName B 2024-11-10T04:38:42,299 DEBUG [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:42,300 INFO [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] regionserver.HStore(327): Store=12628841f5cc772abac56da27a91cfca/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:38:42,300 INFO [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,301 INFO [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12628841f5cc772abac56da27a91cfca columnFamilyName C 2024-11-10T04:38:42,301 DEBUG [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:38:42,301 INFO [StoreOpener-12628841f5cc772abac56da27a91cfca-1 {}] regionserver.HStore(327): Store=12628841f5cc772abac56da27a91cfca/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:38:42,301 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,302 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,302 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,303 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,303 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,303 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:38:42,304 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,305 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:38:42,306 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 12628841f5cc772abac56da27a91cfca; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61769546, jitterRate=-0.07956203818321228}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:38:42,306 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:42,306 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 12628841f5cc772abac56da27a91cfca: Running coprocessor pre-open hook at 1731213522296Writing region info on filesystem at 1731213522296Initializing all the Stores at 1731213522297 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213522297Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213522297Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213522297Cleaning up temporary data from old regions at 1731213522303 (+6 ms)Running coprocessor post-open hooks at 1731213522306 (+3 ms)Region opened successfully at 1731213522306 2024-11-10T04:38:42,307 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., pid=126, masterSystemTime=1731213522292 2024-11-10T04:38:42,309 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:42,309 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:42,309 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=12628841f5cc772abac56da27a91cfca, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:42,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:38:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-10T04:38:42,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T04:38:42,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 in 170 msec 2024-11-10T04:38:42,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-10T04:38:42,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12628841f5cc772abac56da27a91cfca, ASSIGN in 325 msec 2024-11-10T04:38:42,315 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:38:42,315 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213522315"}]},"ts":"1731213522315"} 2024-11-10T04:38:42,316 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T04:38:42,316 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:38:42,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-10T04:38:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-10T04:38:43,322 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:43,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,324 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e436efb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,325 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161b7999, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,326 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,328 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:38:43,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,42929,1731213391190 2024-11-10T04:38:43,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a724e16 2024-11-10T04:38:43,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:38:43,329 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:38:43,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,331 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24c1e4df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,332 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46da7262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,334 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb3f8c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,334 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f5ff5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,336 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150e9828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,337 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277ba79a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,338 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f5f922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39af654c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,341 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a93d26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,342 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40d72ce2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,344 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b7b249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,345 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eb09de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,346 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,346 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,346 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3216b291, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,347 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695643ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,349 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35daf355, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,350 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@619cf36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,351 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,352 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f90ea09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,353 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b50c402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,354 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:38:43,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:38:43,354 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:38:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:38:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:38:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d0f9fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:38:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:38:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:38:43,355 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:38:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8fa22a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:38:43,356 DEBUG [Thread-1875 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,356 DEBUG [Thread-1868 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,356 DEBUG [Thread-1864 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,356 DEBUG [Thread-1879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,356 DEBUG [Thread-1877 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,357 DEBUG [Thread-1862 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,358 DEBUG [Thread-1866 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,358 DEBUG [Thread-1860 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,359 DEBUG [Thread-1871 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,360 DEBUG [Thread-1873 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:38:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:43,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:38:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:38:43,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:38:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-10T04:38:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T04:38:43,366 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2] 2024-11-10T04:38:43,367 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:43,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:43,369 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:43,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:38:43,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/b6932c14506b4ba1a6229ef7ea06f1d7 is 50, key is test_row_0/A:col10/1731213523368/Put/seqid=0 2024-11-10T04:38:43,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742235_1411 (size=14341) 2024-11-10T04:38:43,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213583405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213583406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213583407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213583409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213583410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T04:38:43,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T04:38:43,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:43,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:43,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:43,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213583521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213583521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213583521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213583521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213583531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T04:38:43,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:43,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T04:38:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213583741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213583741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213583742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213583742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213583751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:43,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/b6932c14506b4ba1a6229ef7ea06f1d7 2024-11-10T04:38:43,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/136899dbf0644a5cbbbe9be4e4a19a94 is 50, key is test_row_0/B:col10/1731213523368/Put/seqid=0 2024-11-10T04:38:43,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742236_1412 (size=12001) 2024-11-10T04:38:43,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T04:38:43,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:43,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:43,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:43,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T04:38:43,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:43,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:43,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:43,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T04:38:44,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213584061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213584062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213584062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213584062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213584062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T04:38:44,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:44,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:44,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:44,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:44,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:44,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/136899dbf0644a5cbbbe9be4e4a19a94 2024-11-10T04:38:44,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/618443032681411395ad284fabf86161 is 50, key is test_row_0/C:col10/1731213523368/Put/seqid=0 2024-11-10T04:38:44,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742237_1413 (size=12001) 2024-11-10T04:38:44,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/618443032681411395ad284fabf86161 2024-11-10T04:38:44,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/b6932c14506b4ba1a6229ef7ea06f1d7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b6932c14506b4ba1a6229ef7ea06f1d7 2024-11-10T04:38:44,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b6932c14506b4ba1a6229ef7ea06f1d7, entries=200, sequenceid=12, filesize=14.0 K 2024-11-10T04:38:44,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/136899dbf0644a5cbbbe9be4e4a19a94 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/136899dbf0644a5cbbbe9be4e4a19a94 2024-11-10T04:38:44,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/136899dbf0644a5cbbbe9be4e4a19a94, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T04:38:44,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/618443032681411395ad284fabf86161 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/618443032681411395ad284fabf86161 2024-11-10T04:38:44,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/618443032681411395ad284fabf86161, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T04:38:44,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 12628841f5cc772abac56da27a91cfca in 899ms, sequenceid=12, compaction requested=false 2024-11-10T04:38:44,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:44,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T04:38:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:44,288 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:38:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/be3eace75a774a948b2fd8b29ae5a419 is 50, key is test_row_0/A:col10/1731213523396/Put/seqid=0 2024-11-10T04:38:44,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742238_1414 (size=12001) 2024-11-10T04:38:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T04:38:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:44,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:44,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213584578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213584579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213584580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213584582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213584583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213584691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213584691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,699 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/be3eace75a774a948b2fd8b29ae5a419 2024-11-10T04:38:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/cc6235ebdc4e4fc8bbe875334a7d8d86 is 50, key is test_row_0/B:col10/1731213523396/Put/seqid=0 2024-11-10T04:38:44,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213584701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213584701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213584701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742239_1415 (size=12001) 2024-11-10T04:38:44,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213584911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213584912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213584921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213584921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213584921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,112 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/cc6235ebdc4e4fc8bbe875334a7d8d86 2024-11-10T04:38:45,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/6e6618c4648b4265970fb38f70084f36 is 50, key is test_row_0/C:col10/1731213523396/Put/seqid=0 2024-11-10T04:38:45,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742240_1416 (size=12001) 2024-11-10T04:38:45,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213585221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213585222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213585231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213585231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213585231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T04:38:45,530 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/6e6618c4648b4265970fb38f70084f36 2024-11-10T04:38:45,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/be3eace75a774a948b2fd8b29ae5a419 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/be3eace75a774a948b2fd8b29ae5a419 2024-11-10T04:38:45,539 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/be3eace75a774a948b2fd8b29ae5a419, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T04:38:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/cc6235ebdc4e4fc8bbe875334a7d8d86 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/cc6235ebdc4e4fc8bbe875334a7d8d86 2024-11-10T04:38:45,543 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/cc6235ebdc4e4fc8bbe875334a7d8d86, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T04:38:45,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/6e6618c4648b4265970fb38f70084f36 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/6e6618c4648b4265970fb38f70084f36 2024-11-10T04:38:45,547 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/6e6618c4648b4265970fb38f70084f36, entries=150, sequenceid=37, filesize=11.7 K 2024-11-10T04:38:45,548 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 12628841f5cc772abac56da27a91cfca in 1260ms, sequenceid=37, compaction requested=false 2024-11-10T04:38:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-10T04:38:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-10T04:38:45,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-10T04:38:45,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1830 sec 2024-11-10T04:38:45,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1910 sec 2024-11-10T04:38:45,744 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:45,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:38:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:45,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/f028d51d7d0a43e08b0776380d105984 is 50, key is test_row_0/A:col10/1731213524577/Put/seqid=0 2024-11-10T04:38:45,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742241_1417 (size=14341) 2024-11-10T04:38:45,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213585808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213585809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213585810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213585811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213585815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213585931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213585931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213585931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213585931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213585931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/f028d51d7d0a43e08b0776380d105984 2024-11-10T04:38:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213586151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213586151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213586151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213586151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213586152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/d87da2f3e92a48a9b965b43b931002e8 is 50, key is test_row_0/B:col10/1731213524577/Put/seqid=0 2024-11-10T04:38:46,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742242_1418 (size=12001) 2024-11-10T04:38:46,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/d87da2f3e92a48a9b965b43b931002e8 2024-11-10T04:38:46,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/154af490592448c39fcf97e6e02988e2 is 50, key is test_row_0/C:col10/1731213524577/Put/seqid=0 2024-11-10T04:38:46,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742243_1419 (size=12001) 2024-11-10T04:38:46,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/154af490592448c39fcf97e6e02988e2 2024-11-10T04:38:46,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/f028d51d7d0a43e08b0776380d105984 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/f028d51d7d0a43e08b0776380d105984 2024-11-10T04:38:46,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/f028d51d7d0a43e08b0776380d105984, entries=200, sequenceid=49, filesize=14.0 K 2024-11-10T04:38:46,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/d87da2f3e92a48a9b965b43b931002e8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d87da2f3e92a48a9b965b43b931002e8 2024-11-10T04:38:46,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d87da2f3e92a48a9b965b43b931002e8, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T04:38:46,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/154af490592448c39fcf97e6e02988e2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/154af490592448c39fcf97e6e02988e2 2024-11-10T04:38:46,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/154af490592448c39fcf97e6e02988e2, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T04:38:46,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 12628841f5cc772abac56da27a91cfca in 450ms, sequenceid=49, compaction requested=true 2024-11-10T04:38:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:46,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:46,194 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:46,194 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:46,195 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:46,195 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:46,195 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/A is initiating minor compaction (all files) 2024-11-10T04:38:46,195 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/B is initiating minor compaction (all files) 2024-11-10T04:38:46,195 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/A in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:46,195 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/B in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:46,195 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b6932c14506b4ba1a6229ef7ea06f1d7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/be3eace75a774a948b2fd8b29ae5a419, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/f028d51d7d0a43e08b0776380d105984] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=39.7 K 2024-11-10T04:38:46,195 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/136899dbf0644a5cbbbe9be4e4a19a94, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/cc6235ebdc4e4fc8bbe875334a7d8d86, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d87da2f3e92a48a9b965b43b931002e8] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=35.2 K 2024-11-10T04:38:46,195 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6932c14506b4ba1a6229ef7ea06f1d7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731213523366 2024-11-10T04:38:46,195 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 136899dbf0644a5cbbbe9be4e4a19a94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731213523366 2024-11-10T04:38:46,196 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting be3eace75a774a948b2fd8b29ae5a419, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731213523396 2024-11-10T04:38:46,196 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6235ebdc4e4fc8bbe875334a7d8d86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731213523396 2024-11-10T04:38:46,196 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f028d51d7d0a43e08b0776380d105984, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731213524577 2024-11-10T04:38:46,196 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d87da2f3e92a48a9b965b43b931002e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731213524577 2024-11-10T04:38:46,202 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#B#compaction#351 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:46,203 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/65869e3b5896493fafea0f94a8ecf0fc is 50, key is test_row_0/B:col10/1731213524577/Put/seqid=0 2024-11-10T04:38:46,205 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#A#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:46,205 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/68ac281cb8644f37923285891d998f8b is 50, key is test_row_0/A:col10/1731213524577/Put/seqid=0 2024-11-10T04:38:46,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742244_1420 (size=12104) 2024-11-10T04:38:46,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742245_1421 (size=12104) 2024-11-10T04:38:46,211 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/65869e3b5896493fafea0f94a8ecf0fc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/65869e3b5896493fafea0f94a8ecf0fc 2024-11-10T04:38:46,215 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12628841f5cc772abac56da27a91cfca/B of 12628841f5cc772abac56da27a91cfca into 65869e3b5896493fafea0f94a8ecf0fc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:46,215 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:46,215 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/B, priority=13, startTime=1731213526194; duration=0sec 2024-11-10T04:38:46,215 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:46,215 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:B 2024-11-10T04:38:46,215 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:46,219 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:46,219 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/C is initiating minor compaction (all files) 2024-11-10T04:38:46,219 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/C in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:46,219 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/618443032681411395ad284fabf86161, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/6e6618c4648b4265970fb38f70084f36, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/154af490592448c39fcf97e6e02988e2] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=35.2 K 2024-11-10T04:38:46,219 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 618443032681411395ad284fabf86161, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731213523366 2024-11-10T04:38:46,219 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e6618c4648b4265970fb38f70084f36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731213523396 2024-11-10T04:38:46,220 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 154af490592448c39fcf97e6e02988e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731213524577 2024-11-10T04:38:46,225 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:46,225 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/52681ff290dd4a2caa637f281bafc1e0 is 50, key is test_row_0/C:col10/1731213524577/Put/seqid=0 2024-11-10T04:38:46,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742246_1422 (size=12104) 2024-11-10T04:38:46,231 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T04:38:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:46,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:46,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:38:46,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/d867e17b455348fe946f32d6cffebb57 is 50, key is test_row_0/A:col10/1731213525809/Put/seqid=0 2024-11-10T04:38:46,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742247_1423 (size=14341) 2024-11-10T04:38:46,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213586482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213586483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213586484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213586488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213586489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213586601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213586601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213586601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,614 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/68ac281cb8644f37923285891d998f8b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/68ac281cb8644f37923285891d998f8b 2024-11-10T04:38:46,618 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12628841f5cc772abac56da27a91cfca/A of 12628841f5cc772abac56da27a91cfca into 68ac281cb8644f37923285891d998f8b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:46,618 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:46,618 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/A, priority=13, startTime=1731213526194; duration=0sec 2024-11-10T04:38:46,618 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:46,618 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:A 2024-11-10T04:38:46,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213586611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213586611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,634 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/52681ff290dd4a2caa637f281bafc1e0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/52681ff290dd4a2caa637f281bafc1e0 2024-11-10T04:38:46,638 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12628841f5cc772abac56da27a91cfca/C of 12628841f5cc772abac56da27a91cfca into 52681ff290dd4a2caa637f281bafc1e0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:46,638 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:46,638 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/C, priority=13, startTime=1731213526194; duration=0sec 2024-11-10T04:38:46,638 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:46,638 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:C 2024-11-10T04:38:46,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213586811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213586811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213586811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213586831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213586831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:46,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/d867e17b455348fe946f32d6cffebb57 2024-11-10T04:38:46,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/d7947331e5a340e4bd5ff431107802fe is 50, key is test_row_0/B:col10/1731213525809/Put/seqid=0 2024-11-10T04:38:46,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742248_1424 (size=12001) 2024-11-10T04:38:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213587121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213587121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213587121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213587151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213587151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/d7947331e5a340e4bd5ff431107802fe 2024-11-10T04:38:47,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/efec107f6c1e44af93cb426213333520 is 50, key is test_row_0/C:col10/1731213525809/Put/seqid=0 2024-11-10T04:38:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742249_1425 (size=12001) 2024-11-10T04:38:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T04:38:47,531 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:47,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-10T04:38:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T04:38:47,534 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:47,535 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:47,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T04:38:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213587641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213587641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213587641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213587660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213587671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:47,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T04:38:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:47,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:47,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/efec107f6c1e44af93cb426213333520 2024-11-10T04:38:47,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/d867e17b455348fe946f32d6cffebb57 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d867e17b455348fe946f32d6cffebb57 2024-11-10T04:38:47,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d867e17b455348fe946f32d6cffebb57, entries=200, sequenceid=75, filesize=14.0 K 2024-11-10T04:38:47,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/d7947331e5a340e4bd5ff431107802fe as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d7947331e5a340e4bd5ff431107802fe 2024-11-10T04:38:47,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d7947331e5a340e4bd5ff431107802fe, entries=150, sequenceid=75, filesize=11.7 K 2024-11-10T04:38:47,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/efec107f6c1e44af93cb426213333520 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/efec107f6c1e44af93cb426213333520 2024-11-10T04:38:47,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/efec107f6c1e44af93cb426213333520, entries=150, sequenceid=75, filesize=11.7 K 2024-11-10T04:38:47,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 12628841f5cc772abac56da27a91cfca in 1251ms, sequenceid=75, compaction requested=false 2024-11-10T04:38:47,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:47,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T04:38:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:47,840 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:38:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/09981a6e0bd44efab1fecf4ee8e6b8e9 is 50, key is test_row_0/A:col10/1731213526483/Put/seqid=0 2024-11-10T04:38:47,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742250_1426 (size=12001) 2024-11-10T04:38:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T04:38:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T04:38:48,249 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/09981a6e0bd44efab1fecf4ee8e6b8e9 2024-11-10T04:38:48,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/b39bae8a1b694f3fb180fe5b7ad98cb6 is 50, key is test_row_0/B:col10/1731213526483/Put/seqid=0 2024-11-10T04:38:48,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742251_1427 (size=12001) 2024-11-10T04:38:48,664 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/b39bae8a1b694f3fb180fe5b7ad98cb6 2024-11-10T04:38:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:48,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T04:38:48,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/d799dec7bc614ce0b155104dce356855 is 50, key is test_row_0/C:col10/1731213526483/Put/seqid=0 2024-11-10T04:38:48,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742252_1428 (size=12001) 2024-11-10T04:38:48,677 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/d799dec7bc614ce0b155104dce356855 2024-11-10T04:38:48,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/09981a6e0bd44efab1fecf4ee8e6b8e9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/09981a6e0bd44efab1fecf4ee8e6b8e9 2024-11-10T04:38:48,684 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/09981a6e0bd44efab1fecf4ee8e6b8e9, entries=150, sequenceid=88, filesize=11.7 K 2024-11-10T04:38:48,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/b39bae8a1b694f3fb180fe5b7ad98cb6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b39bae8a1b694f3fb180fe5b7ad98cb6 2024-11-10T04:38:48,688 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b39bae8a1b694f3fb180fe5b7ad98cb6, entries=150, sequenceid=88, filesize=11.7 K 2024-11-10T04:38:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/d799dec7bc614ce0b155104dce356855 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d799dec7bc614ce0b155104dce356855 2024-11-10T04:38:48,693 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d799dec7bc614ce0b155104dce356855, entries=150, sequenceid=88, filesize=11.7 K 2024-11-10T04:38:48,693 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=60.38 KB/61830 for 12628841f5cc772abac56da27a91cfca in 853ms, sequenceid=88, compaction requested=true 2024-11-10T04:38:48,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:48,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:48,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-10T04:38:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-10T04:38:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:48,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:48,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T04:38:48,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-10T04:38:48,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1600 sec 2024-11-10T04:38:48,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1660 sec 2024-11-10T04:38:48,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/16de1949d9694d9ab04b83e8b3b76bf6 is 50, key is test_row_0/A:col10/1731213528688/Put/seqid=0 2024-11-10T04:38:48,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742253_1429 (size=14341) 2024-11-10T04:38:48,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213588757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:48,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213588772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213588773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213588774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213588775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:48,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213588882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:48,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213588891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213588891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:48,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213588901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213588901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:48,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213589101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/16de1949d9694d9ab04b83e8b3b76bf6 2024-11-10T04:38:49,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/e79c6d9d392f48db839b74661aa5d55a is 50, key is test_row_0/B:col10/1731213528688/Put/seqid=0 2024-11-10T04:38:49,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742254_1430 (size=12001) 2024-11-10T04:38:49,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213589111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213589111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213589122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213589122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213589411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213589431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213589431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213589442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213589442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/e79c6d9d392f48db839b74661aa5d55a 2024-11-10T04:38:49,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/51269575ce724912b81a2392f4a9730c is 50, key is test_row_0/C:col10/1731213528688/Put/seqid=0 2024-11-10T04:38:49,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742255_1431 (size=12001) 2024-11-10T04:38:49,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/51269575ce724912b81a2392f4a9730c 2024-11-10T04:38:49,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/16de1949d9694d9ab04b83e8b3b76bf6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/16de1949d9694d9ab04b83e8b3b76bf6 2024-11-10T04:38:49,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/16de1949d9694d9ab04b83e8b3b76bf6, entries=200, sequenceid=101, filesize=14.0 K 2024-11-10T04:38:49,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/e79c6d9d392f48db839b74661aa5d55a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e79c6d9d392f48db839b74661aa5d55a 2024-11-10T04:38:49,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e79c6d9d392f48db839b74661aa5d55a, entries=150, sequenceid=101, filesize=11.7 K 2024-11-10T04:38:49,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/51269575ce724912b81a2392f4a9730c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/51269575ce724912b81a2392f4a9730c 2024-11-10T04:38:49,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/51269575ce724912b81a2392f4a9730c, entries=150, sequenceid=101, filesize=11.7 K 2024-11-10T04:38:49,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 12628841f5cc772abac56da27a91cfca in 858ms, sequenceid=101, compaction requested=true 2024-11-10T04:38:49,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:49,553 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:49,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:49,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:49,553 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:49,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:49,554 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:49,554 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/A is initiating minor compaction (all files) 2024-11-10T04:38:49,554 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/A in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:49,554 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/68ac281cb8644f37923285891d998f8b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d867e17b455348fe946f32d6cffebb57, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/09981a6e0bd44efab1fecf4ee8e6b8e9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/16de1949d9694d9ab04b83e8b3b76bf6] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=51.5 K 2024-11-10T04:38:49,555 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:49,555 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/B is initiating minor compaction (all files) 2024-11-10T04:38:49,555 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/B in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:49,555 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ac281cb8644f37923285891d998f8b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731213524577 2024-11-10T04:38:49,555 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/65869e3b5896493fafea0f94a8ecf0fc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d7947331e5a340e4bd5ff431107802fe, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b39bae8a1b694f3fb180fe5b7ad98cb6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e79c6d9d392f48db839b74661aa5d55a] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=47.0 K 2024-11-10T04:38:49,555 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 65869e3b5896493fafea0f94a8ecf0fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731213524577 2024-11-10T04:38:49,555 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d867e17b455348fe946f32d6cffebb57, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731213525807 2024-11-10T04:38:49,556 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09981a6e0bd44efab1fecf4ee8e6b8e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731213526483 2024-11-10T04:38:49,556 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d7947331e5a340e4bd5ff431107802fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731213525807 2024-11-10T04:38:49,556 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16de1949d9694d9ab04b83e8b3b76bf6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731213528688 2024-11-10T04:38:49,556 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b39bae8a1b694f3fb180fe5b7ad98cb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731213526483 2024-11-10T04:38:49,557 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e79c6d9d392f48db839b74661aa5d55a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731213528688 2024-11-10T04:38:49,566 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#A#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:49,566 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#B#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:49,566 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/86d8db4c433a4ebd94a5f87be81508ea is 50, key is test_row_0/A:col10/1731213528688/Put/seqid=0 2024-11-10T04:38:49,567 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/d0f83580536b42c9870053fb0d596793 is 50, key is test_row_0/B:col10/1731213528688/Put/seqid=0 2024-11-10T04:38:49,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742256_1432 (size=12241) 2024-11-10T04:38:49,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742257_1433 (size=12241) 2024-11-10T04:38:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T04:38:49,681 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:49,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-10T04:38:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T04:38:49,683 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:49,684 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:49,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T04:38:49,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T04:38:49,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:49,837 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T04:38:49,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/75c16b5262ee44a7a1dd133217d69478 is 50, key is test_row_0/A:col10/1731213528774/Put/seqid=0 2024-11-10T04:38:49,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742258_1434 (size=12001) 2024-11-10T04:38:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:49,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:49,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213589963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,975 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/86d8db4c433a4ebd94a5f87be81508ea as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/86d8db4c433a4ebd94a5f87be81508ea 2024-11-10T04:38:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213589971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,979 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/A of 12628841f5cc772abac56da27a91cfca into 86d8db4c433a4ebd94a5f87be81508ea(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:49,979 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:49,979 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/A, priority=12, startTime=1731213529553; duration=0sec 2024-11-10T04:38:49,979 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:49,979 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:A 2024-11-10T04:38:49,979 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:49,980 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:49,980 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/C is initiating minor compaction (all files) 2024-11-10T04:38:49,981 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/C in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:49,981 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/52681ff290dd4a2caa637f281bafc1e0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/efec107f6c1e44af93cb426213333520, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d799dec7bc614ce0b155104dce356855, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/51269575ce724912b81a2392f4a9730c] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=47.0 K 2024-11-10T04:38:49,981 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52681ff290dd4a2caa637f281bafc1e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731213524577 2024-11-10T04:38:49,981 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting efec107f6c1e44af93cb426213333520, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731213525807 2024-11-10T04:38:49,981 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d799dec7bc614ce0b155104dce356855, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731213526483 2024-11-10T04:38:49,982 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51269575ce724912b81a2392f4a9730c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731213528688 2024-11-10T04:38:49,983 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/d0f83580536b42c9870053fb0d596793 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d0f83580536b42c9870053fb0d596793 2024-11-10T04:38:49,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213589973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213589974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213589975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:49,990 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#C#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:49,990 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/B of 12628841f5cc772abac56da27a91cfca into d0f83580536b42c9870053fb0d596793(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:49,990 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:49,990 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/B, priority=12, startTime=1731213529553; duration=0sec 2024-11-10T04:38:49,990 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:49,990 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:B 2024-11-10T04:38:49,990 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/86c23b80c83d48a2a6f54513c0af2394 is 50, key is test_row_0/C:col10/1731213528688/Put/seqid=0 2024-11-10T04:38:50,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742259_1435 (size=12241) 2024-11-10T04:38:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T04:38:50,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213590081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213590081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213590091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213590091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213590101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,246 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/75c16b5262ee44a7a1dd133217d69478 2024-11-10T04:38:50,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/b0e66d9d7b1d4902a2b32e1b15d8a62b is 50, key is test_row_0/B:col10/1731213528774/Put/seqid=0 2024-11-10T04:38:50,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742260_1436 (size=12001) 2024-11-10T04:38:50,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213590291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213590301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213590301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T04:38:50,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213590311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213590321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,404 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/86c23b80c83d48a2a6f54513c0af2394 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/86c23b80c83d48a2a6f54513c0af2394 2024-11-10T04:38:50,408 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/C of 12628841f5cc772abac56da27a91cfca into 86c23b80c83d48a2a6f54513c0af2394(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:50,408 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:50,408 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/C, priority=12, startTime=1731213529554; duration=0sec 2024-11-10T04:38:50,408 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:50,408 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:C 2024-11-10T04:38:50,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213590611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213590621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213590621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213590631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213590631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:50,658 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/b0e66d9d7b1d4902a2b32e1b15d8a62b 2024-11-10T04:38:50,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/43a6bc29c4e74494b048834a6e2c3847 is 50, key is test_row_0/C:col10/1731213528774/Put/seqid=0 2024-11-10T04:38:50,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742261_1437 (size=12001) 2024-11-10T04:38:50,684 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/43a6bc29c4e74494b048834a6e2c3847 2024-11-10T04:38:50,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/75c16b5262ee44a7a1dd133217d69478 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/75c16b5262ee44a7a1dd133217d69478 2024-11-10T04:38:50,695 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/75c16b5262ee44a7a1dd133217d69478, entries=150, sequenceid=124, filesize=11.7 K 2024-11-10T04:38:50,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/b0e66d9d7b1d4902a2b32e1b15d8a62b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b0e66d9d7b1d4902a2b32e1b15d8a62b 2024-11-10T04:38:50,702 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b0e66d9d7b1d4902a2b32e1b15d8a62b, entries=150, sequenceid=124, filesize=11.7 K 2024-11-10T04:38:50,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/43a6bc29c4e74494b048834a6e2c3847 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/43a6bc29c4e74494b048834a6e2c3847 2024-11-10T04:38:50,706 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/43a6bc29c4e74494b048834a6e2c3847, entries=150, sequenceid=124, filesize=11.7 K 2024-11-10T04:38:50,707 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 12628841f5cc772abac56da27a91cfca in 870ms, sequenceid=124, compaction requested=false 2024-11-10T04:38:50,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:50,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:50,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-10T04:38:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-10T04:38:50,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T04:38:50,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-10T04:38:50,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0290 sec 2024-11-10T04:38:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T04:38:50,822 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:50,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-10T04:38:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T04:38:50,824 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:50,825 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:50,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T04:38:50,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T04:38:50,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:50,977 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T04:38:50,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/e4c213b2adf64180bac2b59afe092029 is 50, key is test_row_0/A:col10/1731213529972/Put/seqid=0 2024-11-10T04:38:50,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742262_1438 (size=12151) 2024-11-10T04:38:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:51,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T04:38:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213591173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213591173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213591174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213591175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213591176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213591291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213591291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213591291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213591291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213591301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,385 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/e4c213b2adf64180bac2b59afe092029 2024-11-10T04:38:51,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/484a67b27b9f4e61b82dbdc0a8940558 is 50, key is test_row_0/B:col10/1731213529972/Put/seqid=0 2024-11-10T04:38:51,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742263_1439 (size=12151) 2024-11-10T04:38:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T04:38:51,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213591501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213591501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213591501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213591501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213591522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,584 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T04:38:51,584 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T04:38:51,798 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/484a67b27b9f4e61b82dbdc0a8940558 2024-11-10T04:38:51,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/09a8f1ebbd66459992b8786a98325593 is 50, key is test_row_0/C:col10/1731213529972/Put/seqid=0 2024-11-10T04:38:51,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742264_1440 (size=12151) 2024-11-10T04:38:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213591811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213591821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213591821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213591821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213591841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:51,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T04:38:52,214 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/09a8f1ebbd66459992b8786a98325593 2024-11-10T04:38:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/e4c213b2adf64180bac2b59afe092029 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/e4c213b2adf64180bac2b59afe092029 2024-11-10T04:38:52,230 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/e4c213b2adf64180bac2b59afe092029, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T04:38:52,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/484a67b27b9f4e61b82dbdc0a8940558 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/484a67b27b9f4e61b82dbdc0a8940558 2024-11-10T04:38:52,235 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/484a67b27b9f4e61b82dbdc0a8940558, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T04:38:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/09a8f1ebbd66459992b8786a98325593 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/09a8f1ebbd66459992b8786a98325593 2024-11-10T04:38:52,240 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/09a8f1ebbd66459992b8786a98325593, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T04:38:52,241 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 12628841f5cc772abac56da27a91cfca in 1264ms, sequenceid=140, compaction requested=true 2024-11-10T04:38:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-10T04:38:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-10T04:38:52,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-10T04:38:52,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-10T04:38:52,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4260 sec 2024-11-10T04:38:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:52,337 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:52,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T04:38:52,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/b1c7c097f94245759b7bbd58d1af9539 is 50, key is test_row_0/A:col10/1731213531174/Put/seqid=0 2024-11-10T04:38:52,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742265_1441 (size=12151) 2024-11-10T04:38:52,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/b1c7c097f94245759b7bbd58d1af9539 2024-11-10T04:38:52,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213592366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213592366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/588702d2f20b4ec79f0cde0a4558c510 is 50, key is test_row_0/B:col10/1731213531174/Put/seqid=0 2024-11-10T04:38:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213592368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213592369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213592371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742266_1442 (size=12151) 2024-11-10T04:38:52,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213592491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213592491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213592491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213592491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213592712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213592713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213592713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213592713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:52,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/588702d2f20b4ec79f0cde0a4558c510 2024-11-10T04:38:52,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/f1930c013ccf4743b4b765fcd82a1df6 is 50, key is test_row_0/C:col10/1731213531174/Put/seqid=0 2024-11-10T04:38:52,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742267_1443 (size=12151) 2024-11-10T04:38:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T04:38:52,971 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:52,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-10T04:38:52,974 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:52,975 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:52,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T04:38:53,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213593031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213593032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213593032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213593041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T04:38:53,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T04:38:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:53,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:53,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:53,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/f1930c013ccf4743b4b765fcd82a1df6 2024-11-10T04:38:53,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/b1c7c097f94245759b7bbd58d1af9539 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b1c7c097f94245759b7bbd58d1af9539 2024-11-10T04:38:53,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b1c7c097f94245759b7bbd58d1af9539, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T04:38:53,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/588702d2f20b4ec79f0cde0a4558c510 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/588702d2f20b4ec79f0cde0a4558c510 2024-11-10T04:38:53,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/588702d2f20b4ec79f0cde0a4558c510, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T04:38:53,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/f1930c013ccf4743b4b765fcd82a1df6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f1930c013ccf4743b4b765fcd82a1df6 2024-11-10T04:38:53,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f1930c013ccf4743b4b765fcd82a1df6, entries=150, sequenceid=164, filesize=11.9 K 2024-11-10T04:38:53,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 12628841f5cc772abac56da27a91cfca in 914ms, sequenceid=164, compaction requested=true 2024-11-10T04:38:53,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:53,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:53,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:53,251 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:53,251 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:53,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:53,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:53,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:53,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:53,253 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:53,253 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/A is initiating minor compaction (all files) 2024-11-10T04:38:53,253 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/A in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:53,253 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/86d8db4c433a4ebd94a5f87be81508ea, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/75c16b5262ee44a7a1dd133217d69478, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/e4c213b2adf64180bac2b59afe092029, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b1c7c097f94245759b7bbd58d1af9539] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=47.4 K 2024-11-10T04:38:53,253 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:53,253 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/B is initiating minor compaction (all files) 2024-11-10T04:38:53,253 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/B in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:53,253 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d0f83580536b42c9870053fb0d596793, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b0e66d9d7b1d4902a2b32e1b15d8a62b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/484a67b27b9f4e61b82dbdc0a8940558, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/588702d2f20b4ec79f0cde0a4558c510] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=47.4 K 2024-11-10T04:38:53,254 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f83580536b42c9870053fb0d596793, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731213528688 2024-11-10T04:38:53,254 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86d8db4c433a4ebd94a5f87be81508ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731213528688 2024-11-10T04:38:53,255 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b0e66d9d7b1d4902a2b32e1b15d8a62b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731213528770 2024-11-10T04:38:53,256 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c16b5262ee44a7a1dd133217d69478, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731213528770 2024-11-10T04:38:53,257 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 484a67b27b9f4e61b82dbdc0a8940558, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731213529958 2024-11-10T04:38:53,257 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c213b2adf64180bac2b59afe092029, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731213529958 2024-11-10T04:38:53,258 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 588702d2f20b4ec79f0cde0a4558c510, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731213531174 2024-11-10T04:38:53,259 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c7c097f94245759b7bbd58d1af9539, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731213531174 2024-11-10T04:38:53,272 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:53,272 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#B#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:53,272 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5be029b56dac45fba88ed09cdd111045 is 50, key is test_row_0/B:col10/1731213531174/Put/seqid=0 2024-11-10T04:38:53,272 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/2b969219bd3e491785cb6433c843f011 is 50, key is test_row_0/A:col10/1731213531174/Put/seqid=0 2024-11-10T04:38:53,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T04:38:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:53,282 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:38:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742268_1444 (size=12527) 2024-11-10T04:38:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T04:38:53,295 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5be029b56dac45fba88ed09cdd111045 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5be029b56dac45fba88ed09cdd111045 2024-11-10T04:38:53,299 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/B of 12628841f5cc772abac56da27a91cfca into 5be029b56dac45fba88ed09cdd111045(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:53,299 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:53,299 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/B, priority=12, startTime=1731213533251; duration=0sec 2024-11-10T04:38:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/0c4864897bcb4d839d6ff9863d1d94f0 is 50, key is test_row_0/A:col10/1731213532369/Put/seqid=0 2024-11-10T04:38:53,301 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:53,301 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:B 2024-11-10T04:38:53,301 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:53,304 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:53,304 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/C is initiating minor compaction (all files) 2024-11-10T04:38:53,304 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/C in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:53,304 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/86c23b80c83d48a2a6f54513c0af2394, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/43a6bc29c4e74494b048834a6e2c3847, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/09a8f1ebbd66459992b8786a98325593, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f1930c013ccf4743b4b765fcd82a1df6] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=47.4 K 2024-11-10T04:38:53,305 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c23b80c83d48a2a6f54513c0af2394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731213528688 2024-11-10T04:38:53,305 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a6bc29c4e74494b048834a6e2c3847, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731213528770 2024-11-10T04:38:53,305 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 09a8f1ebbd66459992b8786a98325593, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731213529958 2024-11-10T04:38:53,305 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f1930c013ccf4743b4b765fcd82a1df6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731213531174 2024-11-10T04:38:53,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742269_1445 (size=12527) 2024-11-10T04:38:53,353 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#C#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:53,353 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/4a59ae753a714d5395bb943119335eb5 is 50, key is test_row_0/C:col10/1731213531174/Put/seqid=0 2024-11-10T04:38:53,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742270_1446 (size=12151) 2024-11-10T04:38:53,360 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/0c4864897bcb4d839d6ff9863d1d94f0 2024-11-10T04:38:53,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742271_1447 (size=12527) 2024-11-10T04:38:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/999c578b9e9b4af895936011a17bbd53 is 50, key is test_row_0/B:col10/1731213532369/Put/seqid=0 2024-11-10T04:38:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:53,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:53,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742272_1448 (size=12151) 2024-11-10T04:38:53,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213593564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213593565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213593567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213593577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213593579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T04:38:53,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213593681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213593692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213593692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213593701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213593701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,739 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/2b969219bd3e491785cb6433c843f011 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/2b969219bd3e491785cb6433c843f011 2024-11-10T04:38:53,744 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/A of 12628841f5cc772abac56da27a91cfca into 2b969219bd3e491785cb6433c843f011(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:53,744 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:53,744 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/A, priority=12, startTime=1731213533251; duration=0sec 2024-11-10T04:38:53,744 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:53,744 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:A 2024-11-10T04:38:53,786 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/4a59ae753a714d5395bb943119335eb5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/4a59ae753a714d5395bb943119335eb5 2024-11-10T04:38:53,792 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/C of 12628841f5cc772abac56da27a91cfca into 4a59ae753a714d5395bb943119335eb5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:53,792 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:53,792 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/C, priority=12, startTime=1731213533251; duration=0sec 2024-11-10T04:38:53,792 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:53,792 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:C 2024-11-10T04:38:53,807 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/999c578b9e9b4af895936011a17bbd53 2024-11-10T04:38:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/61076af8233f401d9db5e4f1c1861561 is 50, key is test_row_0/C:col10/1731213532369/Put/seqid=0 2024-11-10T04:38:53,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742273_1449 (size=12151) 2024-11-10T04:38:53,833 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/61076af8233f401d9db5e4f1c1861561 2024-11-10T04:38:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/0c4864897bcb4d839d6ff9863d1d94f0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/0c4864897bcb4d839d6ff9863d1d94f0 2024-11-10T04:38:53,842 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/0c4864897bcb4d839d6ff9863d1d94f0, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T04:38:53,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/999c578b9e9b4af895936011a17bbd53 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/999c578b9e9b4af895936011a17bbd53 2024-11-10T04:38:53,846 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/999c578b9e9b4af895936011a17bbd53, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T04:38:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/61076af8233f401d9db5e4f1c1861561 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/61076af8233f401d9db5e4f1c1861561 2024-11-10T04:38:53,851 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/61076af8233f401d9db5e4f1c1861561, entries=150, sequenceid=176, filesize=11.9 K 2024-11-10T04:38:53,852 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 12628841f5cc772abac56da27a91cfca in 570ms, sequenceid=176, compaction requested=false 2024-11-10T04:38:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:53,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-10T04:38:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-10T04:38:53,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T04:38:53,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 882 msec 2024-11-10T04:38:53,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 887 msec 2024-11-10T04:38:53,900 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:53,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:38:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:53,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/48a55612fb7e4a7b9fde0aba9a4a3c3e is 50, key is test_row_0/A:col10/1731213533577/Put/seqid=0 2024-11-10T04:38:53,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213593927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213593929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213593934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213593934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213593935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:53,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742274_1450 (size=14541) 2024-11-10T04:38:53,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/48a55612fb7e4a7b9fde0aba9a4a3c3e 2024-11-10T04:38:53,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/478a78265a964a3c86346ffd49150200 is 50, key is test_row_0/B:col10/1731213533577/Put/seqid=0 2024-11-10T04:38:54,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742275_1451 (size=12151) 2024-11-10T04:38:54,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/478a78265a964a3c86346ffd49150200 2024-11-10T04:38:54,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/f3a1d904361d4e8c886851baaad5a74f is 50, key is test_row_0/C:col10/1731213533577/Put/seqid=0 2024-11-10T04:38:54,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742276_1452 (size=12151) 2024-11-10T04:38:54,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213594041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213594051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213594051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213594051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213594051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T04:38:54,111 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:54,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-10T04:38:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T04:38:54,115 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:54,116 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:54,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T04:38:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213594251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213594261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213594261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213594261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:54,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213594271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/f3a1d904361d4e8c886851baaad5a74f 2024-11-10T04:38:54,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:54,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/48a55612fb7e4a7b9fde0aba9a4a3c3e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/48a55612fb7e4a7b9fde0aba9a4a3c3e 2024-11-10T04:38:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T04:38:54,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/48a55612fb7e4a7b9fde0aba9a4a3c3e, entries=200, sequenceid=204, filesize=14.2 K 2024-11-10T04:38:54,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/478a78265a964a3c86346ffd49150200 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/478a78265a964a3c86346ffd49150200 2024-11-10T04:38:54,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/478a78265a964a3c86346ffd49150200, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T04:38:54,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/f3a1d904361d4e8c886851baaad5a74f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f3a1d904361d4e8c886851baaad5a74f 2024-11-10T04:38:54,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f3a1d904361d4e8c886851baaad5a74f, entries=150, sequenceid=204, filesize=11.9 K 2024-11-10T04:38:54,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 12628841f5cc772abac56da27a91cfca in 548ms, sequenceid=204, compaction requested=true 2024-11-10T04:38:54,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:54,448 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:54,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:54,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:54,449 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:54,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:54,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:54,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:54,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:54,450 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:54,450 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:54,450 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/B is initiating minor compaction (all files) 2024-11-10T04:38:54,450 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/A is initiating minor compaction (all files) 2024-11-10T04:38:54,450 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/B in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,450 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/A in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,450 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/2b969219bd3e491785cb6433c843f011, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/0c4864897bcb4d839d6ff9863d1d94f0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/48a55612fb7e4a7b9fde0aba9a4a3c3e] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=38.3 K 2024-11-10T04:38:54,450 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5be029b56dac45fba88ed09cdd111045, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/999c578b9e9b4af895936011a17bbd53, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/478a78265a964a3c86346ffd49150200] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=36.0 K 2024-11-10T04:38:54,451 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5be029b56dac45fba88ed09cdd111045, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731213531174 2024-11-10T04:38:54,451 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b969219bd3e491785cb6433c843f011, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731213531174 2024-11-10T04:38:54,451 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 999c578b9e9b4af895936011a17bbd53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731213532348 2024-11-10T04:38:54,452 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c4864897bcb4d839d6ff9863d1d94f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731213532348 2024-11-10T04:38:54,452 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 478a78265a964a3c86346ffd49150200, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731213533563 2024-11-10T04:38:54,452 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a55612fb7e4a7b9fde0aba9a4a3c3e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731213533563 2024-11-10T04:38:54,462 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:54,463 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/c64bdabcd24048a2a0201e62141864b8 is 50, key is test_row_0/B:col10/1731213533577/Put/seqid=0 2024-11-10T04:38:54,469 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:54,470 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/fe9c9a2fd5be47e2b4a61bc9a01937b5 is 50, key is test_row_0/A:col10/1731213533577/Put/seqid=0 2024-11-10T04:38:54,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742277_1453 (size=12629) 2024-11-10T04:38:54,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742278_1454 (size=12629) 2024-11-10T04:38:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:54,571 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:54,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:38:54,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/49ca656bcfd149cd828668d6a3147697 is 50, key is test_row_0/A:col10/1731213534567/Put/seqid=0 2024-11-10T04:38:54,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:54,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:54,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742279_1455 (size=14541) 2024-11-10T04:38:54,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213594625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213594630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213594664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213594664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213594664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:54,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:54,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T04:38:54,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213594771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213594771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213594782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213594782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213594782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,886 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/c64bdabcd24048a2a0201e62141864b8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c64bdabcd24048a2a0201e62141864b8 2024-11-10T04:38:54,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:54,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:54,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:54,891 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12628841f5cc772abac56da27a91cfca/B of 12628841f5cc772abac56da27a91cfca into c64bdabcd24048a2a0201e62141864b8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:54,891 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:54,891 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/B, priority=13, startTime=1731213534448; duration=0sec 2024-11-10T04:38:54,891 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:54,892 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:B 2024-11-10T04:38:54,892 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:38:54,893 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:38:54,893 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/C is initiating minor compaction (all files) 2024-11-10T04:38:54,893 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/C in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:54,893 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/4a59ae753a714d5395bb943119335eb5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/61076af8233f401d9db5e4f1c1861561, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f3a1d904361d4e8c886851baaad5a74f] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=36.0 K 2024-11-10T04:38:54,893 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a59ae753a714d5395bb943119335eb5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731213531174 2024-11-10T04:38:54,894 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 61076af8233f401d9db5e4f1c1861561, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731213532348 2024-11-10T04:38:54,894 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f3a1d904361d4e8c886851baaad5a74f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731213533563 2024-11-10T04:38:54,898 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/fe9c9a2fd5be47e2b4a61bc9a01937b5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/fe9c9a2fd5be47e2b4a61bc9a01937b5 2024-11-10T04:38:54,906 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:54,906 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/47244779dacf44b88154e828e9abce9f is 50, key is test_row_0/C:col10/1731213533577/Put/seqid=0 2024-11-10T04:38:54,907 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12628841f5cc772abac56da27a91cfca/A of 12628841f5cc772abac56da27a91cfca into fe9c9a2fd5be47e2b4a61bc9a01937b5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:54,907 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:54,907 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/A, priority=13, startTime=1731213534448; duration=0sec 2024-11-10T04:38:54,907 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:54,907 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:A 2024-11-10T04:38:54,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742280_1456 (size=12629) 2024-11-10T04:38:54,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/49ca656bcfd149cd828668d6a3147697 2024-11-10T04:38:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213594992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213594992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/908eebb126334dcf89fac3a3c9c55400 is 50, key is test_row_0/B:col10/1731213534567/Put/seqid=0 2024-11-10T04:38:55,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742281_1457 (size=12151) 2024-11-10T04:38:55,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213595001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213595001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213595001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:55,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:55,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T04:38:55,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213595301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,319 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/47244779dacf44b88154e828e9abce9f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/47244779dacf44b88154e828e9abce9f 2024-11-10T04:38:55,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213595311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,328 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12628841f5cc772abac56da27a91cfca/C of 12628841f5cc772abac56da27a91cfca into 47244779dacf44b88154e828e9abce9f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:55,328 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:55,328 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/C, priority=13, startTime=1731213534449; duration=0sec 2024-11-10T04:38:55,328 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:55,328 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:C 2024-11-10T04:38:55,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213595321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213595321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213595321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/908eebb126334dcf89fac3a3c9c55400 2024-11-10T04:38:55,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/544593ee0398460cbb3fd4955f1a13ba is 50, key is test_row_0/C:col10/1731213534567/Put/seqid=0 2024-11-10T04:38:55,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742282_1458 (size=12151) 2024-11-10T04:38:55,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:55,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/544593ee0398460cbb3fd4955f1a13ba 2024-11-10T04:38:55,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/49ca656bcfd149cd828668d6a3147697 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/49ca656bcfd149cd828668d6a3147697 2024-11-10T04:38:55,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/49ca656bcfd149cd828668d6a3147697, entries=200, sequenceid=216, filesize=14.2 K 2024-11-10T04:38:55,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/908eebb126334dcf89fac3a3c9c55400 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/908eebb126334dcf89fac3a3c9c55400 2024-11-10T04:38:55,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/908eebb126334dcf89fac3a3c9c55400, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T04:38:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213595821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/544593ee0398460cbb3fd4955f1a13ba as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/544593ee0398460cbb3fd4955f1a13ba 2024-11-10T04:38:55,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/544593ee0398460cbb3fd4955f1a13ba, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T04:38:55,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213595831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213595842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213595842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213595842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:55,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 12628841f5cc772abac56da27a91cfca in 1284ms, sequenceid=216, compaction requested=false 2024-11-10T04:38:55,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:55,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T04:38:55,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:55,964 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:38:55,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/9597b81d0add425587673f15da116260 is 50, key is test_row_0/A:col10/1731213534638/Put/seqid=0 2024-11-10T04:38:56,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742283_1459 (size=12151) 2024-11-10T04:38:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T04:38:56,434 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/9597b81d0add425587673f15da116260 2024-11-10T04:38:56,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/868b86497adf45ba9e986783c248dd18 is 50, key is test_row_0/B:col10/1731213534638/Put/seqid=0 2024-11-10T04:38:56,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742284_1460 (size=12151) 2024-11-10T04:38:56,459 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/868b86497adf45ba9e986783c248dd18 2024-11-10T04:38:56,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/9d3d4e600f1f456f878616b1cc98e95f is 50, key is test_row_0/C:col10/1731213534638/Put/seqid=0 2024-11-10T04:38:56,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742285_1461 (size=12151) 2024-11-10T04:38:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:56,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:56,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213596866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:56,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213596868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:56,876 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/9d3d4e600f1f456f878616b1cc98e95f 2024-11-10T04:38:56,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/9597b81d0add425587673f15da116260 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9597b81d0add425587673f15da116260 2024-11-10T04:38:56,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213596869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:56,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213596870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213596871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:56,883 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9597b81d0add425587673f15da116260, entries=150, sequenceid=244, filesize=11.9 K 2024-11-10T04:38:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:56,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/868b86497adf45ba9e986783c248dd18 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/868b86497adf45ba9e986783c248dd18 2024-11-10T04:38:56,886 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/868b86497adf45ba9e986783c248dd18, entries=150, sequenceid=244, filesize=11.9 K 2024-11-10T04:38:56,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/9d3d4e600f1f456f878616b1cc98e95f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/9d3d4e600f1f456f878616b1cc98e95f 2024-11-10T04:38:56,890 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/9d3d4e600f1f456f878616b1cc98e95f, entries=150, sequenceid=244, filesize=11.9 K 2024-11-10T04:38:56,891 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 12628841f5cc772abac56da27a91cfca in 926ms, sequenceid=244, compaction requested=true 2024-11-10T04:38:56,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:56,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:56,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-10T04:38:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-10T04:38:56,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-10T04:38:56,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7760 sec 2024-11-10T04:38:56,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.7830 sec 2024-11-10T04:38:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:56,988 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:56,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:38:56,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/4d3613fb3b5c497480c12d959860d5e1 is 50, key is test_row_0/A:col10/1731213536987/Put/seqid=0 2024-11-10T04:38:56,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742286_1462 (size=14541) 2024-11-10T04:38:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213597027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213597028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213597035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213597035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213597041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213597151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213597151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213597151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213597151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213597151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213597371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213597371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213597371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213597371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213597372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/4d3613fb3b5c497480c12d959860d5e1 2024-11-10T04:38:57,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5120729edac4400e8bf6183962c664bf is 50, key is test_row_0/B:col10/1731213536987/Put/seqid=0 2024-11-10T04:38:57,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742287_1463 (size=12151) 2024-11-10T04:38:57,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5120729edac4400e8bf6183962c664bf 2024-11-10T04:38:57,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/8230b475dddf462d81a9ff211c117c8f is 50, key is test_row_0/C:col10/1731213536987/Put/seqid=0 2024-11-10T04:38:57,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742288_1464 (size=12151) 2024-11-10T04:38:57,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/8230b475dddf462d81a9ff211c117c8f 2024-11-10T04:38:57,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/4d3613fb3b5c497480c12d959860d5e1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/4d3613fb3b5c497480c12d959860d5e1 2024-11-10T04:38:57,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/4d3613fb3b5c497480c12d959860d5e1, entries=200, sequenceid=256, filesize=14.2 K 2024-11-10T04:38:57,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5120729edac4400e8bf6183962c664bf as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5120729edac4400e8bf6183962c664bf 2024-11-10T04:38:57,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5120729edac4400e8bf6183962c664bf, entries=150, sequenceid=256, filesize=11.9 K 2024-11-10T04:38:57,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/8230b475dddf462d81a9ff211c117c8f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/8230b475dddf462d81a9ff211c117c8f 2024-11-10T04:38:57,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/8230b475dddf462d81a9ff211c117c8f, entries=150, sequenceid=256, filesize=11.9 K 2024-11-10T04:38:57,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 12628841f5cc772abac56da27a91cfca in 450ms, sequenceid=256, compaction requested=true 2024-11-10T04:38:57,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:57,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:38:57,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:57,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:38:57,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:57,438 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:57,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:38:57,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:57,438 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:57,439 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:57,439 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/B is initiating minor compaction (all files) 2024-11-10T04:38:57,439 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/B in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:57,440 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c64bdabcd24048a2a0201e62141864b8, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/908eebb126334dcf89fac3a3c9c55400, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/868b86497adf45ba9e986783c248dd18, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5120729edac4400e8bf6183962c664bf] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=47.9 K 2024-11-10T04:38:57,440 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:57,440 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/A is initiating minor compaction (all files) 2024-11-10T04:38:57,440 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/A in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:57,440 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/fe9c9a2fd5be47e2b4a61bc9a01937b5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/49ca656bcfd149cd828668d6a3147697, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9597b81d0add425587673f15da116260, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/4d3613fb3b5c497480c12d959860d5e1] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=52.6 K 2024-11-10T04:38:57,440 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c64bdabcd24048a2a0201e62141864b8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731213533563 2024-11-10T04:38:57,440 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe9c9a2fd5be47e2b4a61bc9a01937b5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731213533563 2024-11-10T04:38:57,441 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 908eebb126334dcf89fac3a3c9c55400, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731213533932 2024-11-10T04:38:57,441 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49ca656bcfd149cd828668d6a3147697, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731213533932 2024-11-10T04:38:57,441 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9597b81d0add425587673f15da116260, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731213534614 2024-11-10T04:38:57,441 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 868b86497adf45ba9e986783c248dd18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731213534614 2024-11-10T04:38:57,441 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5120729edac4400e8bf6183962c664bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731213536869 2024-11-10T04:38:57,441 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3613fb3b5c497480c12d959860d5e1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731213536865 2024-11-10T04:38:57,454 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#A#compaction#396 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:57,454 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/9ef05dfbd1ae43fbb1f818487afff1a4 is 50, key is test_row_0/A:col10/1731213536987/Put/seqid=0 2024-11-10T04:38:57,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742289_1465 (size=12765) 2024-11-10T04:38:57,471 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#B#compaction#397 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:57,471 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/b7a0e080e4d44861925987db07d6b3c1 is 50, key is test_row_0/B:col10/1731213536987/Put/seqid=0 2024-11-10T04:38:57,475 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/9ef05dfbd1ae43fbb1f818487afff1a4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9ef05dfbd1ae43fbb1f818487afff1a4 2024-11-10T04:38:57,479 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/A of 12628841f5cc772abac56da27a91cfca into 9ef05dfbd1ae43fbb1f818487afff1a4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:57,480 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:57,480 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/A, priority=12, startTime=1731213537438; duration=0sec 2024-11-10T04:38:57,480 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:38:57,480 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:A 2024-11-10T04:38:57,480 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:38:57,481 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:38:57,481 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/C is initiating minor compaction (all files) 2024-11-10T04:38:57,481 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/C in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:57,481 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/47244779dacf44b88154e828e9abce9f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/544593ee0398460cbb3fd4955f1a13ba, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/9d3d4e600f1f456f878616b1cc98e95f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/8230b475dddf462d81a9ff211c117c8f] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=47.9 K 2024-11-10T04:38:57,482 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47244779dacf44b88154e828e9abce9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731213533563 2024-11-10T04:38:57,482 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 544593ee0398460cbb3fd4955f1a13ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731213533932 2024-11-10T04:38:57,483 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d3d4e600f1f456f878616b1cc98e95f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731213534614 2024-11-10T04:38:57,483 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8230b475dddf462d81a9ff211c117c8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731213536869 2024-11-10T04:38:57,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742290_1466 (size=12765) 2024-11-10T04:38:57,502 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#C#compaction#398 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:38:57,503 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/e38d10a6624143fb9fd8b721e6ce8fe3 is 50, key is test_row_0/C:col10/1731213536987/Put/seqid=0 2024-11-10T04:38:57,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742291_1467 (size=12765) 2024-11-10T04:38:57,520 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/e38d10a6624143fb9fd8b721e6ce8fe3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/e38d10a6624143fb9fd8b721e6ce8fe3 2024-11-10T04:38:57,529 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/C of 12628841f5cc772abac56da27a91cfca into e38d10a6624143fb9fd8b721e6ce8fe3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:57,529 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:57,529 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/C, priority=12, startTime=1731213537438; duration=0sec 2024-11-10T04:38:57,529 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:57,529 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:C 2024-11-10T04:38:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:57,698 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:57,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:38:57,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/61c57f8287684b14b48c6a45b5b62098 is 50, key is test_row_0/A:col10/1731213537021/Put/seqid=0 2024-11-10T04:38:57,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742292_1468 (size=14741) 2024-11-10T04:38:57,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213597713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213597713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213597714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213597714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213597714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213597831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213597831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213597831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213597832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213597832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:57,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:57,891 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/b7a0e080e4d44861925987db07d6b3c1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b7a0e080e4d44861925987db07d6b3c1 2024-11-10T04:38:57,900 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/B of 12628841f5cc772abac56da27a91cfca into b7a0e080e4d44861925987db07d6b3c1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:38:57,900 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:57,900 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/B, priority=12, startTime=1731213537438; duration=0sec 2024-11-10T04:38:57,900 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:38:57,900 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:B 2024-11-10T04:38:58,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213598051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213598051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213598052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213598052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213598052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/61c57f8287684b14b48c6a45b5b62098 2024-11-10T04:38:58,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5656b81d799b424cb63231ab3f549331 is 50, key is test_row_0/B:col10/1731213537021/Put/seqid=0 2024-11-10T04:38:58,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742293_1469 (size=12301) 2024-11-10T04:38:58,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5656b81d799b424cb63231ab3f549331 2024-11-10T04:38:58,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/68421bf39f6c45b59b3a94c7a57182bc is 50, key is test_row_0/C:col10/1731213537021/Put/seqid=0 2024-11-10T04:38:58,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742294_1470 (size=12301) 2024-11-10T04:38:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T04:38:58,282 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:58,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-10T04:38:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T04:38:58,285 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:58,285 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:58,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213598371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213598371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213598371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213598372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213598372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T04:38:58,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T04:38:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:58,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:58,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/68421bf39f6c45b59b3a94c7a57182bc 2024-11-10T04:38:58,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/61c57f8287684b14b48c6a45b5b62098 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/61c57f8287684b14b48c6a45b5b62098 2024-11-10T04:38:58,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/61c57f8287684b14b48c6a45b5b62098, entries=200, sequenceid=283, filesize=14.4 K 2024-11-10T04:38:58,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5656b81d799b424cb63231ab3f549331 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5656b81d799b424cb63231ab3f549331 2024-11-10T04:38:58,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5656b81d799b424cb63231ab3f549331, entries=150, sequenceid=283, filesize=12.0 K 2024-11-10T04:38:58,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/68421bf39f6c45b59b3a94c7a57182bc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/68421bf39f6c45b59b3a94c7a57182bc 2024-11-10T04:38:58,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/68421bf39f6c45b59b3a94c7a57182bc, entries=150, sequenceid=283, filesize=12.0 K 2024-11-10T04:38:58,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 12628841f5cc772abac56da27a91cfca in 868ms, sequenceid=283, compaction requested=false 2024-11-10T04:38:58,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:58,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-10T04:38:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:58,593 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:38:58,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/03204773d82b4191aa96c6c9d0bfbd22 is 50, key is test_row_0/A:col10/1731213537709/Put/seqid=0 2024-11-10T04:38:58,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742295_1471 (size=12301) 2024-11-10T04:38:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T04:38:58,603 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/03204773d82b4191aa96c6c9d0bfbd22 2024-11-10T04:38:58,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/a617387db1ec48c49c8699f099abbe72 is 50, key is test_row_0/B:col10/1731213537709/Put/seqid=0 2024-11-10T04:38:58,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742296_1472 (size=12301) 2024-11-10T04:38:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:58,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T04:38:58,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213598925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213598926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213598927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213598928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213598935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,012 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/a617387db1ec48c49c8699f099abbe72 2024-11-10T04:38:59,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/7b40d9ab057b434c84d2a47107a01668 is 50, key is test_row_0/C:col10/1731213537709/Put/seqid=0 2024-11-10T04:38:59,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742297_1473 (size=12301) 2024-11-10T04:38:59,027 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/7b40d9ab057b434c84d2a47107a01668 2024-11-10T04:38:59,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/03204773d82b4191aa96c6c9d0bfbd22 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/03204773d82b4191aa96c6c9d0bfbd22 2024-11-10T04:38:59,035 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/03204773d82b4191aa96c6c9d0bfbd22, entries=150, sequenceid=296, filesize=12.0 K 2024-11-10T04:38:59,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/a617387db1ec48c49c8699f099abbe72 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/a617387db1ec48c49c8699f099abbe72 2024-11-10T04:38:59,039 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/a617387db1ec48c49c8699f099abbe72, entries=150, sequenceid=296, filesize=12.0 K 2024-11-10T04:38:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/7b40d9ab057b434c84d2a47107a01668 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/7b40d9ab057b434c84d2a47107a01668 2024-11-10T04:38:59,045 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/7b40d9ab057b434c84d2a47107a01668, entries=150, sequenceid=296, filesize=12.0 K 2024-11-10T04:38:59,046 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 12628841f5cc772abac56da27a91cfca in 453ms, sequenceid=296, compaction requested=true 2024-11-10T04:38:59,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:38:59,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:59,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-10T04:38:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-10T04:38:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:38:59,048 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:38:59,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:38:59,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-10T04:38:59,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 761 msec 2024-11-10T04:38:59,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 768 msec 2024-11-10T04:38:59,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/362368451dfc4c25a69bb6c3bf12f073 is 50, key is test_row_0/A:col10/1731213539046/Put/seqid=0 2024-11-10T04:38:59,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742298_1474 (size=14741) 2024-11-10T04:38:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213599065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213599065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213599066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213599081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213599083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213599191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213599191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213599191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213599201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213599202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213599411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213599411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213599411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213599411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213599411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T04:38:59,422 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:38:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:38:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-10T04:38:59,425 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:38:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T04:38:59,426 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:38:59,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:38:59,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/362368451dfc4c25a69bb6c3bf12f073 2024-11-10T04:38:59,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/c261273ba33d4abbbff9f311194019f3 is 50, key is test_row_0/B:col10/1731213539046/Put/seqid=0 2024-11-10T04:38:59,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742299_1475 (size=12301) 2024-11-10T04:38:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T04:38:59,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-10T04:38:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:59,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:59,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-10T04:38:59,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:59,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:59,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:59,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:59,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T04:38:59,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213599732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213599732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213599732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213599732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:38:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213599732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:38:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:38:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:38:59,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/c261273ba33d4abbbff9f311194019f3 2024-11-10T04:38:59,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/d6fb78dd14424e6fa28869c609dde85c is 50, key is test_row_0/C:col10/1731213539046/Put/seqid=0 2024-11-10T04:38:59,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-10T04:38:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:38:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:38:59,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:38:59,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742300_1476 (size=12301) 2024-11-10T04:39:00,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-10T04:39:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:39:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:00,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:00,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T04:39:00,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-10T04:39:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:39:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:00,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:00,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213600251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:00,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213600251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:00,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213600252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:00,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213600252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213600252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:00,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/d6fb78dd14424e6fa28869c609dde85c 2024-11-10T04:39:00,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/362368451dfc4c25a69bb6c3bf12f073 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/362368451dfc4c25a69bb6c3bf12f073 2024-11-10T04:39:00,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/362368451dfc4c25a69bb6c3bf12f073, entries=200, sequenceid=321, filesize=14.4 K 2024-11-10T04:39:00,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/c261273ba33d4abbbff9f311194019f3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c261273ba33d4abbbff9f311194019f3 2024-11-10T04:39:00,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c261273ba33d4abbbff9f311194019f3, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T04:39:00,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/d6fb78dd14424e6fa28869c609dde85c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d6fb78dd14424e6fa28869c609dde85c 2024-11-10T04:39:00,306 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:39:00,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d6fb78dd14424e6fa28869c609dde85c, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T04:39:00,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 12628841f5cc772abac56da27a91cfca in 1260ms, sequenceid=321, compaction requested=true 2024-11-10T04:39:00,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:00,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:00,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:00,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:00,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:00,309 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:39:00,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:00,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:00,309 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:39:00,310 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54548 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:39:00,310 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:39:00,310 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/A is initiating minor compaction (all files) 2024-11-10T04:39:00,310 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/B is initiating minor compaction (all files) 2024-11-10T04:39:00,310 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/B in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:00,310 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/A in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:00,310 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b7a0e080e4d44861925987db07d6b3c1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5656b81d799b424cb63231ab3f549331, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/a617387db1ec48c49c8699f099abbe72, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c261273ba33d4abbbff9f311194019f3] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=48.5 K 2024-11-10T04:39:00,310 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9ef05dfbd1ae43fbb1f818487afff1a4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/61c57f8287684b14b48c6a45b5b62098, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/03204773d82b4191aa96c6c9d0bfbd22, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/362368451dfc4c25a69bb6c3bf12f073] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=53.3 K 2024-11-10T04:39:00,311 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a0e080e4d44861925987db07d6b3c1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731213536869 2024-11-10T04:39:00,311 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef05dfbd1ae43fbb1f818487afff1a4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731213536869 2024-11-10T04:39:00,311 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5656b81d799b424cb63231ab3f549331, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731213537021 2024-11-10T04:39:00,311 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61c57f8287684b14b48c6a45b5b62098, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731213537021 2024-11-10T04:39:00,311 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a617387db1ec48c49c8699f099abbe72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731213537709 2024-11-10T04:39:00,312 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03204773d82b4191aa96c6c9d0bfbd22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731213537709 2024-11-10T04:39:00,312 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c261273ba33d4abbbff9f311194019f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731213538926 2024-11-10T04:39:00,312 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 362368451dfc4c25a69bb6c3bf12f073, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731213538926 2024-11-10T04:39:00,320 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:00,320 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/9d1fb2c5c0314826a9c1f5244afeb7ee is 50, key is test_row_0/B:col10/1731213539046/Put/seqid=0 2024-11-10T04:39:00,323 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:00,323 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/700b2500fa014393b70a5bee931c29ba is 50, key is test_row_0/A:col10/1731213539046/Put/seqid=0 2024-11-10T04:39:00,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742301_1477 (size=13051) 2024-11-10T04:39:00,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742302_1478 (size=13051) 2024-11-10T04:39:00,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-10T04:39:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:00,346 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:39:00,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/7e8fe9a0a05548919ae6c62968f4b288 is 50, key is test_row_0/A:col10/1731213539081/Put/seqid=0 2024-11-10T04:39:00,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742303_1479 (size=12301) 2024-11-10T04:39:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T04:39:00,742 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/9d1fb2c5c0314826a9c1f5244afeb7ee as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/9d1fb2c5c0314826a9c1f5244afeb7ee 2024-11-10T04:39:00,742 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/700b2500fa014393b70a5bee931c29ba as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/700b2500fa014393b70a5bee931c29ba 2024-11-10T04:39:00,749 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/B of 12628841f5cc772abac56da27a91cfca into 9d1fb2c5c0314826a9c1f5244afeb7ee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:00,749 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/A of 12628841f5cc772abac56da27a91cfca into 700b2500fa014393b70a5bee931c29ba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:00,749 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:00,749 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:00,749 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/B, priority=12, startTime=1731213540309; duration=0sec 2024-11-10T04:39:00,749 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/A, priority=12, startTime=1731213540309; duration=0sec 2024-11-10T04:39:00,749 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:00,749 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:B 2024-11-10T04:39:00,749 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:00,749 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:A 2024-11-10T04:39:00,749 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T04:39:00,750 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T04:39:00,751 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/C is initiating minor compaction (all files) 2024-11-10T04:39:00,751 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/C in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:00,751 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/e38d10a6624143fb9fd8b721e6ce8fe3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/68421bf39f6c45b59b3a94c7a57182bc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/7b40d9ab057b434c84d2a47107a01668, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d6fb78dd14424e6fa28869c609dde85c] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=48.5 K 2024-11-10T04:39:00,751 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e38d10a6624143fb9fd8b721e6ce8fe3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731213536869 2024-11-10T04:39:00,751 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 68421bf39f6c45b59b3a94c7a57182bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731213537021 2024-11-10T04:39:00,752 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b40d9ab057b434c84d2a47107a01668, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731213537709 2024-11-10T04:39:00,752 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d6fb78dd14424e6fa28869c609dde85c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731213538926 2024-11-10T04:39:00,754 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/7e8fe9a0a05548919ae6c62968f4b288 2024-11-10T04:39:00,761 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:00,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/dcb77d6163204252b6512eeeaa9020bd is 50, key is test_row_0/B:col10/1731213539081/Put/seqid=0 2024-11-10T04:39:00,762 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/5c286bc812384822bed270e4ea968565 is 50, key is test_row_0/C:col10/1731213539046/Put/seqid=0 2024-11-10T04:39:00,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742304_1480 (size=12301) 2024-11-10T04:39:00,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742305_1481 (size=13051) 2024-11-10T04:39:01,173 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/dcb77d6163204252b6512eeeaa9020bd 2024-11-10T04:39:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/57768e801b25402da822bc2946e89b83 is 50, key is test_row_0/C:col10/1731213539081/Put/seqid=0 2024-11-10T04:39:01,184 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/5c286bc812384822bed270e4ea968565 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/5c286bc812384822bed270e4ea968565 2024-11-10T04:39:01,187 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12628841f5cc772abac56da27a91cfca/C of 12628841f5cc772abac56da27a91cfca into 5c286bc812384822bed270e4ea968565(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:01,187 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:01,187 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/C, priority=12, startTime=1731213540309; duration=0sec 2024-11-10T04:39:01,188 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:01,188 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:C 2024-11-10T04:39:01,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742306_1482 (size=12301) 2024-11-10T04:39:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:39:01,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. as already flushing 2024-11-10T04:39:01,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213601303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213601306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213601310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213601311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213601315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213601421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213601421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213601431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213601431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213601431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T04:39:01,590 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/57768e801b25402da822bc2946e89b83 2024-11-10T04:39:01,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/7e8fe9a0a05548919ae6c62968f4b288 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/7e8fe9a0a05548919ae6c62968f4b288 2024-11-10T04:39:01,597 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/7e8fe9a0a05548919ae6c62968f4b288, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T04:39:01,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/dcb77d6163204252b6512eeeaa9020bd as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/dcb77d6163204252b6512eeeaa9020bd 2024-11-10T04:39:01,601 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/dcb77d6163204252b6512eeeaa9020bd, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T04:39:01,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/57768e801b25402da822bc2946e89b83 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/57768e801b25402da822bc2946e89b83 2024-11-10T04:39:01,604 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/57768e801b25402da822bc2946e89b83, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T04:39:01,605 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 12628841f5cc772abac56da27a91cfca in 1259ms, sequenceid=332, compaction requested=false 2024-11-10T04:39:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-10T04:39:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-10T04:39:01,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-10T04:39:01,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-11-10T04:39:01,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 2.1860 sec 2024-11-10T04:39:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:39:01,637 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:01,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T04:39:01,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/d7b9a9da00e24b66bddebf546ba7423c is 50, key is test_row_0/A:col10/1731213541636/Put/seqid=0 2024-11-10T04:39:01,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742307_1483 (size=14741) 2024-11-10T04:39:01,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213601652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213601656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213601659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213601659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213601661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213601771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213601771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213601771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213601781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213601781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213601981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213601981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213601981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213601991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:01,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213601991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/d7b9a9da00e24b66bddebf546ba7423c 2024-11-10T04:39:02,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/e1d76dabae5449d0b705176d4609fc6f is 50, key is test_row_0/B:col10/1731213541636/Put/seqid=0 2024-11-10T04:39:02,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742308_1484 (size=12301) 2024-11-10T04:39:02,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213602291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213602292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213602292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213602301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213602301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/e1d76dabae5449d0b705176d4609fc6f 2024-11-10T04:39:02,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/64577685876a418291494de475574dfb is 50, key is test_row_0/C:col10/1731213541636/Put/seqid=0 2024-11-10T04:39:02,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742309_1485 (size=12301) 2024-11-10T04:39:02,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1731213602802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54828 deadline: 1731213602802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54836 deadline: 1731213602802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1731213602811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54856 deadline: 1731213602811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 , the old value is region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=2 because the exception is null or not the one we care about 2024-11-10T04:39:02,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/64577685876a418291494de475574dfb 2024-11-10T04:39:02,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/d7b9a9da00e24b66bddebf546ba7423c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d7b9a9da00e24b66bddebf546ba7423c 2024-11-10T04:39:02,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d7b9a9da00e24b66bddebf546ba7423c, entries=200, sequenceid=361, filesize=14.4 K 2024-11-10T04:39:02,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/e1d76dabae5449d0b705176d4609fc6f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e1d76dabae5449d0b705176d4609fc6f 2024-11-10T04:39:02,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e1d76dabae5449d0b705176d4609fc6f, entries=150, sequenceid=361, filesize=12.0 K 2024-11-10T04:39:02,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/64577685876a418291494de475574dfb as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/64577685876a418291494de475574dfb 2024-11-10T04:39:02,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/64577685876a418291494de475574dfb, entries=150, sequenceid=361, filesize=12.0 K 2024-11-10T04:39:02,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 12628841f5cc772abac56da27a91cfca in 1258ms, sequenceid=361, compaction requested=true 2024-11-10T04:39:02,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:02,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:02,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:02,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:02,895 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:02,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:02,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12628841f5cc772abac56da27a91cfca:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:02,896 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:02,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:02,896 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:02,896 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:02,896 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/B is initiating minor compaction (all files) 2024-11-10T04:39:02,896 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/A is initiating minor compaction (all files) 2024-11-10T04:39:02,896 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/B in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:02,896 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/A in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:02,896 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/9d1fb2c5c0314826a9c1f5244afeb7ee, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/dcb77d6163204252b6512eeeaa9020bd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e1d76dabae5449d0b705176d4609fc6f] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=36.8 K 2024-11-10T04:39:02,896 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/700b2500fa014393b70a5bee931c29ba, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/7e8fe9a0a05548919ae6c62968f4b288, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d7b9a9da00e24b66bddebf546ba7423c] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=39.2 K 2024-11-10T04:39:02,897 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d1fb2c5c0314826a9c1f5244afeb7ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731213538926 2024-11-10T04:39:02,897 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 700b2500fa014393b70a5bee931c29ba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731213538926 2024-11-10T04:39:02,897 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb77d6163204252b6512eeeaa9020bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731213539060 2024-11-10T04:39:02,897 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e8fe9a0a05548919ae6c62968f4b288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731213539060 2024-11-10T04:39:02,897 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7b9a9da00e24b66bddebf546ba7423c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731213541296 2024-11-10T04:39:02,897 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e1d76dabae5449d0b705176d4609fc6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731213541304 2024-11-10T04:39:02,903 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:02,904 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5caf41c878084705a5fbcea259820cd8 is 50, key is test_row_0/B:col10/1731213541636/Put/seqid=0 2024-11-10T04:39:02,906 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:02,906 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/abeff1a2dbd34c4981e81f991f0b3a35 is 50, key is test_row_0/A:col10/1731213541636/Put/seqid=0 2024-11-10T04:39:02,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742310_1486 (size=13153) 2024-11-10T04:39:02,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742311_1487 (size=13153) 2024-11-10T04:39:02,912 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/5caf41c878084705a5fbcea259820cd8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5caf41c878084705a5fbcea259820cd8 2024-11-10T04:39:02,915 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/abeff1a2dbd34c4981e81f991f0b3a35 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/abeff1a2dbd34c4981e81f991f0b3a35 2024-11-10T04:39:02,917 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12628841f5cc772abac56da27a91cfca/B of 12628841f5cc772abac56da27a91cfca into 5caf41c878084705a5fbcea259820cd8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:02,917 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:02,917 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/B, priority=13, startTime=1731213542895; duration=0sec 2024-11-10T04:39:02,917 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:02,917 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:B 2024-11-10T04:39:02,917 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:02,918 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:02,918 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 12628841f5cc772abac56da27a91cfca/C is initiating minor compaction (all files) 2024-11-10T04:39:02,918 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12628841f5cc772abac56da27a91cfca/C in TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:02,918 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/5c286bc812384822bed270e4ea968565, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/57768e801b25402da822bc2946e89b83, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/64577685876a418291494de475574dfb] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp, totalSize=36.8 K 2024-11-10T04:39:02,919 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c286bc812384822bed270e4ea968565, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731213538926 2024-11-10T04:39:02,920 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 57768e801b25402da822bc2946e89b83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731213539060 2024-11-10T04:39:02,920 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 64577685876a418291494de475574dfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731213541304 2024-11-10T04:39:02,921 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12628841f5cc772abac56da27a91cfca/A of 12628841f5cc772abac56da27a91cfca into abeff1a2dbd34c4981e81f991f0b3a35(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:02,921 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:02,921 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/A, priority=13, startTime=1731213542895; duration=0sec 2024-11-10T04:39:02,921 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:02,921 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:A 2024-11-10T04:39:02,931 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12628841f5cc772abac56da27a91cfca#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:02,931 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/529cba77c7c546b1a2e4d897d0001570 is 50, key is test_row_0/C:col10/1731213541636/Put/seqid=0 2024-11-10T04:39:02,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742312_1488 (size=13153) 2024-11-10T04:39:03,345 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/529cba77c7c546b1a2e4d897d0001570 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/529cba77c7c546b1a2e4d897d0001570 2024-11-10T04:39:03,351 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12628841f5cc772abac56da27a91cfca/C of 12628841f5cc772abac56da27a91cfca into 529cba77c7c546b1a2e4d897d0001570(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:03,351 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:03,351 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., storeName=12628841f5cc772abac56da27a91cfca/C, priority=13, startTime=1731213542895; duration=0sec 2024-11-10T04:39:03,351 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:03,351 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12628841f5cc772abac56da27a91cfca:C 2024-11-10T04:39:03,361 INFO [Thread-1875 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1875. 2024-11-10T04:39:03,362 DEBUG [Thread-1875 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,362 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,362 INFO [Thread-1873 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1873. 2024-11-10T04:39:03,362 DEBUG [Thread-1873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,362 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,362 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,362 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,364 INFO [Thread-1871 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1871. 2024-11-10T04:39:03,364 DEBUG [Thread-1871 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,364 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,364 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,364 INFO [Thread-1879 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1879. 2024-11-10T04:39:03,364 DEBUG [Thread-1879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,364 DEBUG [Thread-1879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,365 DEBUG [Thread-1879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,365 INFO [Thread-1877 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1877. 2024-11-10T04:39:03,365 DEBUG [Thread-1877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,365 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,365 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T04:39:03,601 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 12628841f5cc772abac56da27a91cfca 2024-11-10T04:39:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:03,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:39:03,812 INFO [Thread-1860 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1860. 2024-11-10T04:39:03,812 DEBUG [Thread-1860 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,813 DEBUG [Thread-1860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,813 DEBUG [Thread-1860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,813 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/d4b3a98f4ede4910890358a0795e5139 is 50, key is test_row_0/A:col10/1731213541659/Put/seqid=0 2024-11-10T04:39:03,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742313_1489 (size=12301) 2024-11-10T04:39:03,823 INFO [Thread-1868 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1868. 2024-11-10T04:39:03,823 DEBUG [Thread-1868 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,823 INFO [Thread-1862 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1862. 2024-11-10T04:39:03,823 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,823 DEBUG [Thread-1862 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,823 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,823 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,823 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,832 INFO [Thread-1864 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1864. 2024-11-10T04:39:03,832 INFO [Thread-1866 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1866. 2024-11-10T04:39:03,832 DEBUG [Thread-1864 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,832 DEBUG [Thread-1866 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:03,832 DEBUG [Thread-1864 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,832 DEBUG [Thread-1866 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,832 DEBUG [Thread-1864 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,832 DEBUG [Thread-1866 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1255 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3765 rows 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1258 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3772 rows 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1257 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3765 rows 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1253 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3756 rows 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1255 2024-11-10T04:39:03,832 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3757 rows 2024-11-10T04:39:03,833 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:39:03,833 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:39:03,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:03,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:03,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T04:39:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T04:39:03,836 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213543836"}]},"ts":"1731213543836"} 2024-11-10T04:39:03,837 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T04:39:03,837 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T04:39:03,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T04:39:03,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12628841f5cc772abac56da27a91cfca, UNASSIGN}] 2024-11-10T04:39:03,840 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12628841f5cc772abac56da27a91cfca, UNASSIGN 2024-11-10T04:39:03,840 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=12628841f5cc772abac56da27a91cfca, regionState=CLOSING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:03,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12628841f5cc772abac56da27a91cfca, UNASSIGN because future has completed 2024-11-10T04:39:03,841 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T04:39:03,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:39:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T04:39:03,993 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 12628841f5cc772abac56da27a91cfca 2024-11-10T04:39:03,993 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T04:39:03,993 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 12628841f5cc772abac56da27a91cfca, disabling compactions & flushes 2024-11-10T04:39:03,994 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T04:39:04,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/d4b3a98f4ede4910890358a0795e5139 2024-11-10T04:39:04,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/fd219c4aa12e42fb826606d30e062ee3 is 50, key is test_row_0/B:col10/1731213541659/Put/seqid=0 2024-11-10T04:39:04,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742314_1490 (size=12301) 2024-11-10T04:39:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T04:39:04,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/fd219c4aa12e42fb826606d30e062ee3 2024-11-10T04:39:04,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/4c363783050b479a9d4b90b314071b4f is 50, key is test_row_0/C:col10/1731213541659/Put/seqid=0 2024-11-10T04:39:04,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742315_1491 (size=12301) 2024-11-10T04:39:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T04:39:05,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/4c363783050b479a9d4b90b314071b4f 2024-11-10T04:39:05,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/d4b3a98f4ede4910890358a0795e5139 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d4b3a98f4ede4910890358a0795e5139 2024-11-10T04:39:05,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d4b3a98f4ede4910890358a0795e5139, entries=150, sequenceid=375, filesize=12.0 K 2024-11-10T04:39:05,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/fd219c4aa12e42fb826606d30e062ee3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/fd219c4aa12e42fb826606d30e062ee3 2024-11-10T04:39:05,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/fd219c4aa12e42fb826606d30e062ee3, entries=150, sequenceid=375, filesize=12.0 K 2024-11-10T04:39:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/4c363783050b479a9d4b90b314071b4f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/4c363783050b479a9d4b90b314071b4f 2024-11-10T04:39:05,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/4c363783050b479a9d4b90b314071b4f, entries=150, sequenceid=375, filesize=12.0 K 2024-11-10T04:39:05,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=26.84 KB/27480 for 12628841f5cc772abac56da27a91cfca in 1237ms, sequenceid=375, compaction requested=false 2024-11-10T04:39:05,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12628841f5cc772abac56da27a91cfca: 2024-11-10T04:39:05,049 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:05,049 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:05,049 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. after waiting 0 ms 2024-11-10T04:39:05,049 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:05,050 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing 12628841f5cc772abac56da27a91cfca 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-10T04:39:05,052 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/30d0340fcdb04bc191dd4d59104bcabc is 50, key is test_row_0/A:col10/1731213543831/Put/seqid=0 2024-11-10T04:39:05,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742316_1492 (size=9857) 2024-11-10T04:39:05,456 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/30d0340fcdb04bc191dd4d59104bcabc 2024-11-10T04:39:05,461 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/3cc4afa4f0dd44fd9aa823b3c56419e1 is 50, key is test_row_0/B:col10/1731213543831/Put/seqid=0 2024-11-10T04:39:05,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742317_1493 (size=9857) 2024-11-10T04:39:05,865 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/3cc4afa4f0dd44fd9aa823b3c56419e1 2024-11-10T04:39:05,870 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/b7e68fa731204f6db89b8927fca21e58 is 50, key is test_row_0/C:col10/1731213543831/Put/seqid=0 2024-11-10T04:39:05,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742318_1494 (size=9857) 2024-11-10T04:39:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T04:39:06,274 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/b7e68fa731204f6db89b8927fca21e58 2024-11-10T04:39:06,277 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/A/30d0340fcdb04bc191dd4d59104bcabc as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/30d0340fcdb04bc191dd4d59104bcabc 2024-11-10T04:39:06,280 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/30d0340fcdb04bc191dd4d59104bcabc, entries=100, sequenceid=382, filesize=9.6 K 2024-11-10T04:39:06,280 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/B/3cc4afa4f0dd44fd9aa823b3c56419e1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/3cc4afa4f0dd44fd9aa823b3c56419e1 2024-11-10T04:39:06,283 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/3cc4afa4f0dd44fd9aa823b3c56419e1, entries=100, sequenceid=382, filesize=9.6 K 2024-11-10T04:39:06,283 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/.tmp/C/b7e68fa731204f6db89b8927fca21e58 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/b7e68fa731204f6db89b8927fca21e58 2024-11-10T04:39:06,286 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/b7e68fa731204f6db89b8927fca21e58, entries=100, sequenceid=382, filesize=9.6 K 2024-11-10T04:39:06,286 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 12628841f5cc772abac56da27a91cfca in 1237ms, sequenceid=382, compaction requested=true 2024-11-10T04:39:06,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b6932c14506b4ba1a6229ef7ea06f1d7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/be3eace75a774a948b2fd8b29ae5a419, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/f028d51d7d0a43e08b0776380d105984, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/68ac281cb8644f37923285891d998f8b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d867e17b455348fe946f32d6cffebb57, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/09981a6e0bd44efab1fecf4ee8e6b8e9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/16de1949d9694d9ab04b83e8b3b76bf6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/86d8db4c433a4ebd94a5f87be81508ea, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/75c16b5262ee44a7a1dd133217d69478, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/e4c213b2adf64180bac2b59afe092029, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/2b969219bd3e491785cb6433c843f011, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b1c7c097f94245759b7bbd58d1af9539, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/0c4864897bcb4d839d6ff9863d1d94f0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/48a55612fb7e4a7b9fde0aba9a4a3c3e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/fe9c9a2fd5be47e2b4a61bc9a01937b5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/49ca656bcfd149cd828668d6a3147697, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9597b81d0add425587673f15da116260, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/4d3613fb3b5c497480c12d959860d5e1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9ef05dfbd1ae43fbb1f818487afff1a4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/61c57f8287684b14b48c6a45b5b62098, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/03204773d82b4191aa96c6c9d0bfbd22, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/362368451dfc4c25a69bb6c3bf12f073, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/700b2500fa014393b70a5bee931c29ba, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/7e8fe9a0a05548919ae6c62968f4b288, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d7b9a9da00e24b66bddebf546ba7423c] to archive 2024-11-10T04:39:06,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:39:06,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b6932c14506b4ba1a6229ef7ea06f1d7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b6932c14506b4ba1a6229ef7ea06f1d7 2024-11-10T04:39:06,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/be3eace75a774a948b2fd8b29ae5a419 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/be3eace75a774a948b2fd8b29ae5a419 2024-11-10T04:39:06,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/f028d51d7d0a43e08b0776380d105984 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/f028d51d7d0a43e08b0776380d105984 2024-11-10T04:39:06,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/68ac281cb8644f37923285891d998f8b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/68ac281cb8644f37923285891d998f8b 2024-11-10T04:39:06,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d867e17b455348fe946f32d6cffebb57 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d867e17b455348fe946f32d6cffebb57 2024-11-10T04:39:06,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/09981a6e0bd44efab1fecf4ee8e6b8e9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/09981a6e0bd44efab1fecf4ee8e6b8e9 2024-11-10T04:39:06,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/16de1949d9694d9ab04b83e8b3b76bf6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/16de1949d9694d9ab04b83e8b3b76bf6 2024-11-10T04:39:06,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/86d8db4c433a4ebd94a5f87be81508ea to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/86d8db4c433a4ebd94a5f87be81508ea 2024-11-10T04:39:06,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/75c16b5262ee44a7a1dd133217d69478 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/75c16b5262ee44a7a1dd133217d69478 2024-11-10T04:39:06,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/e4c213b2adf64180bac2b59afe092029 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/e4c213b2adf64180bac2b59afe092029 2024-11-10T04:39:06,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/2b969219bd3e491785cb6433c843f011 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/2b969219bd3e491785cb6433c843f011 2024-11-10T04:39:06,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b1c7c097f94245759b7bbd58d1af9539 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/b1c7c097f94245759b7bbd58d1af9539 2024-11-10T04:39:06,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/0c4864897bcb4d839d6ff9863d1d94f0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/0c4864897bcb4d839d6ff9863d1d94f0 2024-11-10T04:39:06,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/48a55612fb7e4a7b9fde0aba9a4a3c3e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/48a55612fb7e4a7b9fde0aba9a4a3c3e 2024-11-10T04:39:06,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/fe9c9a2fd5be47e2b4a61bc9a01937b5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/fe9c9a2fd5be47e2b4a61bc9a01937b5 2024-11-10T04:39:06,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/49ca656bcfd149cd828668d6a3147697 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/49ca656bcfd149cd828668d6a3147697 2024-11-10T04:39:06,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9597b81d0add425587673f15da116260 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9597b81d0add425587673f15da116260 2024-11-10T04:39:06,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/4d3613fb3b5c497480c12d959860d5e1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/4d3613fb3b5c497480c12d959860d5e1 2024-11-10T04:39:06,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9ef05dfbd1ae43fbb1f818487afff1a4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/9ef05dfbd1ae43fbb1f818487afff1a4 2024-11-10T04:39:06,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/61c57f8287684b14b48c6a45b5b62098 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/61c57f8287684b14b48c6a45b5b62098 2024-11-10T04:39:06,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/03204773d82b4191aa96c6c9d0bfbd22 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/03204773d82b4191aa96c6c9d0bfbd22 2024-11-10T04:39:06,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/362368451dfc4c25a69bb6c3bf12f073 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/362368451dfc4c25a69bb6c3bf12f073 2024-11-10T04:39:06,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/700b2500fa014393b70a5bee931c29ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/700b2500fa014393b70a5bee931c29ba 2024-11-10T04:39:06,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/7e8fe9a0a05548919ae6c62968f4b288 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/7e8fe9a0a05548919ae6c62968f4b288 2024-11-10T04:39:06,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d7b9a9da00e24b66bddebf546ba7423c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d7b9a9da00e24b66bddebf546ba7423c 2024-11-10T04:39:06,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/136899dbf0644a5cbbbe9be4e4a19a94, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/cc6235ebdc4e4fc8bbe875334a7d8d86, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/65869e3b5896493fafea0f94a8ecf0fc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d87da2f3e92a48a9b965b43b931002e8, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d7947331e5a340e4bd5ff431107802fe, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b39bae8a1b694f3fb180fe5b7ad98cb6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d0f83580536b42c9870053fb0d596793, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e79c6d9d392f48db839b74661aa5d55a, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b0e66d9d7b1d4902a2b32e1b15d8a62b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/484a67b27b9f4e61b82dbdc0a8940558, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5be029b56dac45fba88ed09cdd111045, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/588702d2f20b4ec79f0cde0a4558c510, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/999c578b9e9b4af895936011a17bbd53, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c64bdabcd24048a2a0201e62141864b8, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/478a78265a964a3c86346ffd49150200, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/908eebb126334dcf89fac3a3c9c55400, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/868b86497adf45ba9e986783c248dd18, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b7a0e080e4d44861925987db07d6b3c1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5120729edac4400e8bf6183962c664bf, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5656b81d799b424cb63231ab3f549331, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/a617387db1ec48c49c8699f099abbe72, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/9d1fb2c5c0314826a9c1f5244afeb7ee, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c261273ba33d4abbbff9f311194019f3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/dcb77d6163204252b6512eeeaa9020bd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e1d76dabae5449d0b705176d4609fc6f] to archive 2024-11-10T04:39:06,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:39:06,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/136899dbf0644a5cbbbe9be4e4a19a94 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/136899dbf0644a5cbbbe9be4e4a19a94 2024-11-10T04:39:06,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/cc6235ebdc4e4fc8bbe875334a7d8d86 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/cc6235ebdc4e4fc8bbe875334a7d8d86 2024-11-10T04:39:06,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/65869e3b5896493fafea0f94a8ecf0fc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/65869e3b5896493fafea0f94a8ecf0fc 2024-11-10T04:39:06,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d87da2f3e92a48a9b965b43b931002e8 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d87da2f3e92a48a9b965b43b931002e8 2024-11-10T04:39:06,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d7947331e5a340e4bd5ff431107802fe to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d7947331e5a340e4bd5ff431107802fe 2024-11-10T04:39:06,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b39bae8a1b694f3fb180fe5b7ad98cb6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b39bae8a1b694f3fb180fe5b7ad98cb6 2024-11-10T04:39:06,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d0f83580536b42c9870053fb0d596793 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/d0f83580536b42c9870053fb0d596793 2024-11-10T04:39:06,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e79c6d9d392f48db839b74661aa5d55a to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e79c6d9d392f48db839b74661aa5d55a 2024-11-10T04:39:06,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b0e66d9d7b1d4902a2b32e1b15d8a62b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b0e66d9d7b1d4902a2b32e1b15d8a62b 2024-11-10T04:39:06,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/484a67b27b9f4e61b82dbdc0a8940558 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/484a67b27b9f4e61b82dbdc0a8940558 2024-11-10T04:39:06,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5be029b56dac45fba88ed09cdd111045 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5be029b56dac45fba88ed09cdd111045 2024-11-10T04:39:06,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/588702d2f20b4ec79f0cde0a4558c510 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/588702d2f20b4ec79f0cde0a4558c510 2024-11-10T04:39:06,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/999c578b9e9b4af895936011a17bbd53 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/999c578b9e9b4af895936011a17bbd53 2024-11-10T04:39:06,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c64bdabcd24048a2a0201e62141864b8 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c64bdabcd24048a2a0201e62141864b8 2024-11-10T04:39:06,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/478a78265a964a3c86346ffd49150200 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/478a78265a964a3c86346ffd49150200 2024-11-10T04:39:06,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/908eebb126334dcf89fac3a3c9c55400 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/908eebb126334dcf89fac3a3c9c55400 2024-11-10T04:39:06,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/868b86497adf45ba9e986783c248dd18 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/868b86497adf45ba9e986783c248dd18 2024-11-10T04:39:06,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b7a0e080e4d44861925987db07d6b3c1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/b7a0e080e4d44861925987db07d6b3c1 2024-11-10T04:39:06,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5120729edac4400e8bf6183962c664bf to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5120729edac4400e8bf6183962c664bf 2024-11-10T04:39:06,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5656b81d799b424cb63231ab3f549331 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5656b81d799b424cb63231ab3f549331 2024-11-10T04:39:06,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/a617387db1ec48c49c8699f099abbe72 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/a617387db1ec48c49c8699f099abbe72 2024-11-10T04:39:06,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/9d1fb2c5c0314826a9c1f5244afeb7ee to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/9d1fb2c5c0314826a9c1f5244afeb7ee 2024-11-10T04:39:06,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c261273ba33d4abbbff9f311194019f3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/c261273ba33d4abbbff9f311194019f3 2024-11-10T04:39:06,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/dcb77d6163204252b6512eeeaa9020bd to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/dcb77d6163204252b6512eeeaa9020bd 2024-11-10T04:39:06,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e1d76dabae5449d0b705176d4609fc6f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/e1d76dabae5449d0b705176d4609fc6f 2024-11-10T04:39:06,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/618443032681411395ad284fabf86161, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/6e6618c4648b4265970fb38f70084f36, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/52681ff290dd4a2caa637f281bafc1e0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/154af490592448c39fcf97e6e02988e2, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/efec107f6c1e44af93cb426213333520, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d799dec7bc614ce0b155104dce356855, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/86c23b80c83d48a2a6f54513c0af2394, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/51269575ce724912b81a2392f4a9730c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/43a6bc29c4e74494b048834a6e2c3847, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/09a8f1ebbd66459992b8786a98325593, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/4a59ae753a714d5395bb943119335eb5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f1930c013ccf4743b4b765fcd82a1df6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/61076af8233f401d9db5e4f1c1861561, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/47244779dacf44b88154e828e9abce9f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f3a1d904361d4e8c886851baaad5a74f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/544593ee0398460cbb3fd4955f1a13ba, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/9d3d4e600f1f456f878616b1cc98e95f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/e38d10a6624143fb9fd8b721e6ce8fe3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/8230b475dddf462d81a9ff211c117c8f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/68421bf39f6c45b59b3a94c7a57182bc, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/7b40d9ab057b434c84d2a47107a01668, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/5c286bc812384822bed270e4ea968565, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d6fb78dd14424e6fa28869c609dde85c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/57768e801b25402da822bc2946e89b83, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/64577685876a418291494de475574dfb] to archive 2024-11-10T04:39:06,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:39:06,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/618443032681411395ad284fabf86161 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/618443032681411395ad284fabf86161 2024-11-10T04:39:06,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/6e6618c4648b4265970fb38f70084f36 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/6e6618c4648b4265970fb38f70084f36 2024-11-10T04:39:06,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/52681ff290dd4a2caa637f281bafc1e0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/52681ff290dd4a2caa637f281bafc1e0 2024-11-10T04:39:06,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/154af490592448c39fcf97e6e02988e2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/154af490592448c39fcf97e6e02988e2 2024-11-10T04:39:06,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/efec107f6c1e44af93cb426213333520 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/efec107f6c1e44af93cb426213333520 2024-11-10T04:39:06,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d799dec7bc614ce0b155104dce356855 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d799dec7bc614ce0b155104dce356855 2024-11-10T04:39:06,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/86c23b80c83d48a2a6f54513c0af2394 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/86c23b80c83d48a2a6f54513c0af2394 2024-11-10T04:39:06,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/51269575ce724912b81a2392f4a9730c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/51269575ce724912b81a2392f4a9730c 2024-11-10T04:39:06,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/43a6bc29c4e74494b048834a6e2c3847 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/43a6bc29c4e74494b048834a6e2c3847 2024-11-10T04:39:06,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/09a8f1ebbd66459992b8786a98325593 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/09a8f1ebbd66459992b8786a98325593 2024-11-10T04:39:06,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/4a59ae753a714d5395bb943119335eb5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/4a59ae753a714d5395bb943119335eb5 2024-11-10T04:39:06,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f1930c013ccf4743b4b765fcd82a1df6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f1930c013ccf4743b4b765fcd82a1df6 2024-11-10T04:39:06,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/61076af8233f401d9db5e4f1c1861561 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/61076af8233f401d9db5e4f1c1861561 2024-11-10T04:39:06,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/47244779dacf44b88154e828e9abce9f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/47244779dacf44b88154e828e9abce9f 2024-11-10T04:39:06,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f3a1d904361d4e8c886851baaad5a74f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/f3a1d904361d4e8c886851baaad5a74f 2024-11-10T04:39:06,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/544593ee0398460cbb3fd4955f1a13ba to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/544593ee0398460cbb3fd4955f1a13ba 2024-11-10T04:39:06,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/9d3d4e600f1f456f878616b1cc98e95f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/9d3d4e600f1f456f878616b1cc98e95f 2024-11-10T04:39:06,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/e38d10a6624143fb9fd8b721e6ce8fe3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/e38d10a6624143fb9fd8b721e6ce8fe3 2024-11-10T04:39:06,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/8230b475dddf462d81a9ff211c117c8f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/8230b475dddf462d81a9ff211c117c8f 2024-11-10T04:39:06,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/68421bf39f6c45b59b3a94c7a57182bc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/68421bf39f6c45b59b3a94c7a57182bc 2024-11-10T04:39:06,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/7b40d9ab057b434c84d2a47107a01668 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/7b40d9ab057b434c84d2a47107a01668 2024-11-10T04:39:06,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/5c286bc812384822bed270e4ea968565 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/5c286bc812384822bed270e4ea968565 2024-11-10T04:39:06,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d6fb78dd14424e6fa28869c609dde85c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/d6fb78dd14424e6fa28869c609dde85c 2024-11-10T04:39:06,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/57768e801b25402da822bc2946e89b83 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/57768e801b25402da822bc2946e89b83 2024-11-10T04:39:06,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/64577685876a418291494de475574dfb to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/64577685876a418291494de475574dfb 2024-11-10T04:39:06,355 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/recovered.edits/385.seqid, newMaxSeqId=385, maxSeqId=1 2024-11-10T04:39:06,355 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. 2024-11-10T04:39:06,355 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 12628841f5cc772abac56da27a91cfca: Waiting for close lock at 1731213543993Running coprocessor pre-close hooks at 1731213543993Disabling compacts and flushes for region at 1731213543993Disabling writes for close at 1731213545049 (+1056 ms)Obtaining lock to block concurrent updates at 1731213545050 (+1 ms)Preparing flush snapshotting stores in 12628841f5cc772abac56da27a91cfca at 1731213545050Finished memstore snapshotting TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731213545050Flushing stores of TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca. at 1731213545050Flushing 12628841f5cc772abac56da27a91cfca/A: creating writer at 1731213545050Flushing 12628841f5cc772abac56da27a91cfca/A: appending metadata at 1731213545052 (+2 ms)Flushing 12628841f5cc772abac56da27a91cfca/A: closing flushed file at 1731213545052Flushing 12628841f5cc772abac56da27a91cfca/B: creating writer at 1731213545459 (+407 ms)Flushing 12628841f5cc772abac56da27a91cfca/B: appending metadata at 1731213545461 (+2 ms)Flushing 12628841f5cc772abac56da27a91cfca/B: closing flushed file at 1731213545461Flushing 12628841f5cc772abac56da27a91cfca/C: creating writer at 1731213545868 (+407 ms)Flushing 12628841f5cc772abac56da27a91cfca/C: appending metadata at 1731213545870 (+2 ms)Flushing 12628841f5cc772abac56da27a91cfca/C: closing flushed file at 1731213545870Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37dc4fcd: reopening flushed file at 1731213546277 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4aca8401: reopening flushed file at 1731213546280 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ad74b2b: reopening flushed file at 1731213546283 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 12628841f5cc772abac56da27a91cfca in 1237ms, sequenceid=382, compaction requested=true at 1731213546286 (+3 ms)Writing region close event to WAL at 1731213546352 (+66 ms)Running coprocessor post-close hooks at 1731213546355 (+3 ms)Closed at 1731213546355 2024-11-10T04:39:06,357 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 12628841f5cc772abac56da27a91cfca 2024-11-10T04:39:06,357 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=12628841f5cc772abac56da27a91cfca, regionState=CLOSED 2024-11-10T04:39:06,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:39:06,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-10T04:39:06,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 12628841f5cc772abac56da27a91cfca, server=d1d9e3766cb1,45693,1731213391842 in 2.5180 sec 2024-11-10T04:39:06,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-10T04:39:06,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12628841f5cc772abac56da27a91cfca, UNASSIGN in 2.5220 sec 2024-11-10T04:39:06,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-10T04:39:06,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5250 sec 2024-11-10T04:39:06,365 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213546365"}]},"ts":"1731213546365"} 2024-11-10T04:39:06,366 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T04:39:06,366 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T04:39:06,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5340 sec 2024-11-10T04:39:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T04:39:08,001 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:08,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T04:39:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:08,003 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T04:39:08,004 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:08,006 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca 2024-11-10T04:39:08,007 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/recovered.edits] 2024-11-10T04:39:08,009 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/30d0340fcdb04bc191dd4d59104bcabc to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/30d0340fcdb04bc191dd4d59104bcabc 2024-11-10T04:39:08,010 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/abeff1a2dbd34c4981e81f991f0b3a35 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/abeff1a2dbd34c4981e81f991f0b3a35 2024-11-10T04:39:08,011 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d4b3a98f4ede4910890358a0795e5139 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/A/d4b3a98f4ede4910890358a0795e5139 2024-11-10T04:39:08,013 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/3cc4afa4f0dd44fd9aa823b3c56419e1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/3cc4afa4f0dd44fd9aa823b3c56419e1 2024-11-10T04:39:08,014 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5caf41c878084705a5fbcea259820cd8 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/5caf41c878084705a5fbcea259820cd8 2024-11-10T04:39:08,015 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/fd219c4aa12e42fb826606d30e062ee3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/B/fd219c4aa12e42fb826606d30e062ee3 2024-11-10T04:39:08,017 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/4c363783050b479a9d4b90b314071b4f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/4c363783050b479a9d4b90b314071b4f 2024-11-10T04:39:08,018 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/529cba77c7c546b1a2e4d897d0001570 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/529cba77c7c546b1a2e4d897d0001570 2024-11-10T04:39:08,019 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/b7e68fa731204f6db89b8927fca21e58 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/C/b7e68fa731204f6db89b8927fca21e58 2024-11-10T04:39:08,021 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/recovered.edits/385.seqid to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca/recovered.edits/385.seqid 2024-11-10T04:39:08,021 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/12628841f5cc772abac56da27a91cfca 2024-11-10T04:39:08,021 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T04:39:08,023 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:08,025 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T04:39:08,027 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T04:39:08,027 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:08,027 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T04:39:08,028 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731213548027"}]},"ts":"9223372036854775807"} 2024-11-10T04:39:08,029 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T04:39:08,029 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 12628841f5cc772abac56da27a91cfca, NAME => 'TestAcidGuarantees,,1731213521169.12628841f5cc772abac56da27a91cfca.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T04:39:08,029 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T04:39:08,030 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731213548029"}]},"ts":"9223372036854775807"} 2024-11-10T04:39:08,031 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T04:39:08,031 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:08,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-10T04:39:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T04:39:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T04:39:08,111 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:08,121 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=356 (was 275) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6857 (was 6349) - AvailableMemoryMB LEAK? - 2024-11-10T04:39:08,130 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=356, ProcessCount=11, AvailableMemoryMB=6857 2024-11-10T04:39:08,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T04:39:08,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:39:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:08,133 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:39:08,133 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:08,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-10T04:39:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T04:39:08,134 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:39:08,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742319_1495 (size=959) 2024-11-10T04:39:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T04:39:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T04:39:08,540 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8 2024-11-10T04:39:08,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742320_1496 (size=53) 2024-11-10T04:39:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T04:39:08,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:39:08,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 145f65b29b41c5cf34022ad77be79930, disabling compactions & flushes 2024-11-10T04:39:08,945 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:08,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:08,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. after waiting 0 ms 2024-11-10T04:39:08,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:08,945 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:08,945 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 145f65b29b41c5cf34022ad77be79930: Waiting for close lock at 1731213548945Disabling compacts and flushes for region at 1731213548945Disabling writes for close at 1731213548945Writing region close event to WAL at 1731213548945Closed at 1731213548945 2024-11-10T04:39:08,946 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:39:08,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731213548946"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731213548946"}]},"ts":"1731213548946"} 2024-11-10T04:39:08,947 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:39:08,948 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:39:08,948 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213548948"}]},"ts":"1731213548948"} 2024-11-10T04:39:08,949 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T04:39:08,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, ASSIGN}] 2024-11-10T04:39:08,950 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, ASSIGN 2024-11-10T04:39:08,950 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, ASSIGN; state=OFFLINE, location=d1d9e3766cb1,45693,1731213391842; forceNewPlan=false, retain=false 2024-11-10T04:39:09,101 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=145f65b29b41c5cf34022ad77be79930, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:09,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, ASSIGN because future has completed 2024-11-10T04:39:09,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:39:09,257 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:09,257 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:39:09,257 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,257 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:39:09,257 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,257 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,258 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,259 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 145f65b29b41c5cf34022ad77be79930 columnFamilyName A 2024-11-10T04:39:09,259 DEBUG [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:09,260 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(327): Store=145f65b29b41c5cf34022ad77be79930/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:39:09,260 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,261 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 145f65b29b41c5cf34022ad77be79930 columnFamilyName B 2024-11-10T04:39:09,261 DEBUG [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:09,261 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(327): Store=145f65b29b41c5cf34022ad77be79930/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:39:09,261 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,262 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 145f65b29b41c5cf34022ad77be79930 columnFamilyName C 2024-11-10T04:39:09,262 DEBUG [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:09,262 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(327): Store=145f65b29b41c5cf34022ad77be79930/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:39:09,262 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,263 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,263 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,264 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,264 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,264 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:39:09,265 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,266 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:39:09,266 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened 145f65b29b41c5cf34022ad77be79930; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60389530, jitterRate=-0.10012587904930115}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:39:09,266 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:09,267 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for 145f65b29b41c5cf34022ad77be79930: Running coprocessor pre-open hook at 1731213549257Writing region info on filesystem at 1731213549257Initializing all the Stores at 1731213549258 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213549258Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213549258Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213549258Cleaning up temporary data from old regions at 1731213549264 (+6 ms)Running coprocessor post-open hooks at 1731213549266 (+2 ms)Region opened successfully at 1731213549267 (+1 ms) 2024-11-10T04:39:09,268 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., pid=150, masterSystemTime=1731213549254 2024-11-10T04:39:09,269 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:09,269 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:09,269 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=145f65b29b41c5cf34022ad77be79930, regionState=OPEN, openSeqNum=2, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:09,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:39:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T04:39:09,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-10T04:39:09,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 in 168 msec 2024-11-10T04:39:09,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-10T04:39:09,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, ASSIGN in 324 msec 2024-11-10T04:39:09,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:39:09,275 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213549274"}]},"ts":"1731213549274"} 2024-11-10T04:39:09,276 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T04:39:09,276 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:39:09,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-10T04:39:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T04:39:10,282 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:10,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:10,283 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:10,283 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:10,284 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a8f69a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:10,285 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4cd914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:10,286 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:10,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:10,288 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:39:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is d1d9e3766cb1,42929,1731213391190 2024-11-10T04:39:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a78f656 2024-11-10T04:39:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:39:10,290 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:39:10,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T04:39:10,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:39:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:10,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742321_1497 (size=995) 2024-11-10T04:39:10,703 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T04:39:10,703 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T04:39:10,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T04:39:10,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, REOPEN/MOVE}] 2024-11-10T04:39:10,708 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, REOPEN/MOVE 2024-11-10T04:39:10,708 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=145f65b29b41c5cf34022ad77be79930, regionState=CLOSING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:10,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, REOPEN/MOVE because future has completed 2024-11-10T04:39:10,710 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T04:39:10,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:39:10,862 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:10,862 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T04:39:10,862 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing 145f65b29b41c5cf34022ad77be79930, disabling compactions & flushes 2024-11-10T04:39:10,862 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:10,862 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:10,862 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. after waiting 0 ms 2024-11-10T04:39:10,862 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:10,866 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T04:39:10,866 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:10,866 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for 145f65b29b41c5cf34022ad77be79930: Waiting for close lock at 1731213550862Running coprocessor pre-close hooks at 1731213550862Disabling compacts and flushes for region at 1731213550862Disabling writes for close at 1731213550862Writing region close event to WAL at 1731213550863 (+1 ms)Running coprocessor post-close hooks at 1731213550866 (+3 ms)Closed at 1731213550866 2024-11-10T04:39:10,866 WARN [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: 145f65b29b41c5cf34022ad77be79930 to self. 2024-11-10T04:39:10,868 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:10,868 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=145f65b29b41c5cf34022ad77be79930, regionState=CLOSED 2024-11-10T04:39:10,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:39:10,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-10T04:39:10,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 in 160 msec 2024-11-10T04:39:10,872 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, REOPEN/MOVE; state=CLOSED, location=d1d9e3766cb1,45693,1731213391842; forceNewPlan=false, retain=true 2024-11-10T04:39:11,022 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=145f65b29b41c5cf34022ad77be79930, regionState=OPENING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, REOPEN/MOVE because future has completed 2024-11-10T04:39:11,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:39:11,178 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,178 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:39:11,178 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,178 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:39:11,178 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,178 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,179 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,180 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 145f65b29b41c5cf34022ad77be79930 columnFamilyName A 2024-11-10T04:39:11,181 DEBUG [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:11,181 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(327): Store=145f65b29b41c5cf34022ad77be79930/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:39:11,182 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,182 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 145f65b29b41c5cf34022ad77be79930 columnFamilyName B 2024-11-10T04:39:11,182 DEBUG [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:11,182 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(327): Store=145f65b29b41c5cf34022ad77be79930/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:39:11,182 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,183 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 145f65b29b41c5cf34022ad77be79930 columnFamilyName C 2024-11-10T04:39:11,183 DEBUG [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:11,183 INFO [StoreOpener-145f65b29b41c5cf34022ad77be79930-1 {}] regionserver.HStore(327): Store=145f65b29b41c5cf34022ad77be79930/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:39:11,183 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,184 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,184 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,185 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,185 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,186 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T04:39:11,187 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,187 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened 145f65b29b41c5cf34022ad77be79930; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59304305, jitterRate=-0.11629699170589447}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T04:39:11,187 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,188 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for 145f65b29b41c5cf34022ad77be79930: Running coprocessor pre-open hook at 1731213551178Writing region info on filesystem at 1731213551179 (+1 ms)Initializing all the Stores at 1731213551179Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213551179Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213551179Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731213551179Cleaning up temporary data from old regions at 1731213551185 (+6 ms)Running coprocessor post-open hooks at 1731213551187 (+2 ms)Region opened successfully at 1731213551188 (+1 ms) 2024-11-10T04:39:11,188 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., pid=155, masterSystemTime=1731213551175 2024-11-10T04:39:11,190 DEBUG [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,190 INFO [RS_OPEN_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=145f65b29b41c5cf34022ad77be79930, regionState=OPEN, openSeqNum=5, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:39:11,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-10T04:39:11,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 in 168 msec 2024-11-10T04:39:11,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-10T04:39:11,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, REOPEN/MOVE in 486 msec 2024-11-10T04:39:11,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-10T04:39:11,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-10T04:39:11,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-10T04:39:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-10T04:39:11,199 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:11,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,201 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45a6aa53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,202 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49331a63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,204 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64cb21d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,205 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2223cac8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,207 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@619fb829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f8185d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,209 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4fedd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,210 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43022deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,212 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e923c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,213 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,214 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796154cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,215 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@166b37f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,217 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a9a662f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,218 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58857210, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,219 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6d455a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,221 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75890776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,222 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a39c96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177474e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d1d9e3766cb1,42929,-1 for getting cluster id 2024-11-10T04:39:11,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:39:11,224 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb742aa3-047a-478e-8da4-e4764cdd4b8a' 2024-11-10T04:39:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:39:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb742aa3-047a-478e-8da4-e4764cdd4b8a" 2024-11-10T04:39:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200ac220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d1d9e3766cb1,42929,-1] 2024-11-10T04:39:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:39:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:11,226 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:39:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15849348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:39:11,226 DEBUG [Thread-2251 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,226 DEBUG [Thread-2253 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,226 DEBUG [Thread-2244 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,227 DEBUG [Thread-2236 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,227 DEBUG [Thread-2242 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,227 DEBUG [Thread-2238 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,228 DEBUG [Thread-2240 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,228 DEBUG [Thread-2247 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,228 DEBUG [Thread-2249 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,229 DEBUG [Thread-2240 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,230 DEBUG [Thread-2240 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,230 DEBUG [Thread-2255 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:39:11,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:39:11,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1d9e3766cb1,45693,1731213391842, seqNum=-1] 2024-11-10T04:39:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:39:11,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-10T04:39:11,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:39:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T04:39:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,234 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:39:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,236 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:39:11,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:39:11,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5] 2024-11-10T04:39:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,240 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:11,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:39:11,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213611261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213611262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213611264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213611265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213611265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111046390c78d8a34ead9fec71fc7b2ce000_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_1/A:col10/1731213551239/Put/seqid=0 2024-11-10T04:39:11,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742322_1498 (size=12154) 2024-11-10T04:39:11,286 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:11,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111046390c78d8a34ead9fec71fc7b2ce000_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046390c78d8a34ead9fec71fc7b2ce000_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:11,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/d146b070f2414e89861454f65ce38c9d, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:11,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/d146b070f2414e89861454f65ce38c9d is 175, key is test_row_1/A:col10/1731213551239/Put/seqid=0 2024-11-10T04:39:11,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742323_1499 (size=30951) 2024-11-10T04:39:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T04:39:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213611371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213611371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213611371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213611371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213611371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T04:39:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T04:39:11,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:11,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T04:39:11,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213611581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213611582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213611582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213611582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213611582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T04:39:11,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:11,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/d146b070f2414e89861454f65ce38c9d 2024-11-10T04:39:11,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/e4be4a1810ac4e28a00e5cac666a1041 is 50, key is test_row_1/B:col10/1731213551239/Put/seqid=0 2024-11-10T04:39:11,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742324_1500 (size=9657) 2024-11-10T04:39:11,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T04:39:11,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:11,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:11,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T04:39:11,874 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T04:39:11,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213611891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213611892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213611892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213611892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213611892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:11,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:12,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T04:39:12,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:12,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:12,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:12,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/e4be4a1810ac4e28a00e5cac666a1041 2024-11-10T04:39:12,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/e8bca83d804e41eb99668c2263b73880 is 50, key is test_row_1/C:col10/1731213551239/Put/seqid=0 2024-11-10T04:39:12,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T04:39:12,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:12,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:12,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:12,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742325_1501 (size=9657) 2024-11-10T04:39:12,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T04:39:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:12,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T04:39:12,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213612401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:12,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213612402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:12,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213612402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213612402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:12,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213612411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:12,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T04:39:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:12,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:12,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/e8bca83d804e41eb99668c2263b73880 2024-11-10T04:39:12,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/d146b070f2414e89861454f65ce38c9d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d146b070f2414e89861454f65ce38c9d 2024-11-10T04:39:12,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d146b070f2414e89861454f65ce38c9d, entries=150, sequenceid=16, filesize=30.2 K 2024-11-10T04:39:12,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/e4be4a1810ac4e28a00e5cac666a1041 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e4be4a1810ac4e28a00e5cac666a1041 2024-11-10T04:39:12,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e4be4a1810ac4e28a00e5cac666a1041, entries=100, sequenceid=16, filesize=9.4 K 2024-11-10T04:39:12,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/e8bca83d804e41eb99668c2263b73880 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/e8bca83d804e41eb99668c2263b73880 2024-11-10T04:39:12,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/e8bca83d804e41eb99668c2263b73880, entries=100, sequenceid=16, filesize=9.4 K 2024-11-10T04:39:12,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 145f65b29b41c5cf34022ad77be79930 in 1344ms, sequenceid=16, compaction requested=false 2024-11-10T04:39:12,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:12,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T04:39:12,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:12,615 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T04:39:12,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ba1a0aee982d49b28bfd32b8ebf27e31_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213551257/Put/seqid=0 2024-11-10T04:39:12,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742326_1502 (size=12154) 2024-11-10T04:39:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:12,635 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ba1a0aee982d49b28bfd32b8ebf27e31_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ba1a0aee982d49b28bfd32b8ebf27e31_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3d3da0d7ff1e4777996ab4cd32b3f4d5, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3d3da0d7ff1e4777996ab4cd32b3f4d5 is 175, key is test_row_0/A:col10/1731213551257/Put/seqid=0 2024-11-10T04:39:12,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742327_1503 (size=30955) 2024-11-10T04:39:12,640 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3d3da0d7ff1e4777996ab4cd32b3f4d5 2024-11-10T04:39:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/b9613766772c40ff85b01aae57f97a7f is 50, key is test_row_0/B:col10/1731213551257/Put/seqid=0 2024-11-10T04:39:12,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742328_1504 (size=12001) 2024-11-10T04:39:13,050 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/b9613766772c40ff85b01aae57f97a7f 2024-11-10T04:39:13,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/2a4d5ff345ec4be9a42243d158c1cc43 is 50, key is test_row_0/C:col10/1731213551257/Put/seqid=0 2024-11-10T04:39:13,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742329_1505 (size=12001) 2024-11-10T04:39:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T04:39:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:13,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213613425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213613426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213613427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213613427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213613431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,435 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_1' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,463 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/2a4d5ff345ec4be9a42243d158c1cc43 2024-11-10T04:39:13,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3d3da0d7ff1e4777996ab4cd32b3f4d5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3d3da0d7ff1e4777996ab4cd32b3f4d5 2024-11-10T04:39:13,470 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3d3da0d7ff1e4777996ab4cd32b3f4d5, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T04:39:13,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/b9613766772c40ff85b01aae57f97a7f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/b9613766772c40ff85b01aae57f97a7f 2024-11-10T04:39:13,474 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/b9613766772c40ff85b01aae57f97a7f, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T04:39:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/2a4d5ff345ec4be9a42243d158c1cc43 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2a4d5ff345ec4be9a42243d158c1cc43 2024-11-10T04:39:13,478 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2a4d5ff345ec4be9a42243d158c1cc43, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T04:39:13,478 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 145f65b29b41c5cf34022ad77be79930 in 863ms, sequenceid=40, compaction requested=false 2024-11-10T04:39:13,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:13,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:13,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-10T04:39:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-10T04:39:13,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-10T04:39:13,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2440 sec 2024-11-10T04:39:13,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.2520 sec 2024-11-10T04:39:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:13,544 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:13,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T04:39:13,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dfbceea96eff4490ac14843a8f04e4d6_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213553425/Put/seqid=0 2024-11-10T04:39:13,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742330_1506 (size=17034) 2024-11-10T04:39:13,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:13,559 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dfbceea96eff4490ac14843a8f04e4d6_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dfbceea96eff4490ac14843a8f04e4d6_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:13,560 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/5c592a7ef59f4ff09c41faab00725294, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:13,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/5c592a7ef59f4ff09c41faab00725294 is 175, key is test_row_0/A:col10/1731213553425/Put/seqid=0 2024-11-10T04:39:13,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742331_1507 (size=48139) 2024-11-10T04:39:13,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/5c592a7ef59f4ff09c41faab00725294 2024-11-10T04:39:13,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213613564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213613568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/e97c973643d4425fa27286b20f984101 is 50, key is test_row_0/B:col10/1731213553425/Put/seqid=0 2024-11-10T04:39:13,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213613569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213613570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742332_1508 (size=12001) 2024-11-10T04:39:13,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/e97c973643d4425fa27286b20f984101 2024-11-10T04:39:13,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/ed0f09b5df654078a11453bb6364dfc4 is 50, key is test_row_0/C:col10/1731213553425/Put/seqid=0 2024-11-10T04:39:13,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742333_1509 (size=12001) 2024-11-10T04:39:13,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/ed0f09b5df654078a11453bb6364dfc4 2024-11-10T04:39:13,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/5c592a7ef59f4ff09c41faab00725294 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/5c592a7ef59f4ff09c41faab00725294 2024-11-10T04:39:13,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/5c592a7ef59f4ff09c41faab00725294, entries=250, sequenceid=56, filesize=47.0 K 2024-11-10T04:39:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/e97c973643d4425fa27286b20f984101 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e97c973643d4425fa27286b20f984101 2024-11-10T04:39:13,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e97c973643d4425fa27286b20f984101, entries=150, sequenceid=56, filesize=11.7 K 2024-11-10T04:39:13,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/ed0f09b5df654078a11453bb6364dfc4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ed0f09b5df654078a11453bb6364dfc4 2024-11-10T04:39:13,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ed0f09b5df654078a11453bb6364dfc4, entries=150, sequenceid=56, filesize=11.7 K 2024-11-10T04:39:13,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 145f65b29b41c5cf34022ad77be79930 in 58ms, sequenceid=56, compaction requested=true 2024-11-10T04:39:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:13,602 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:13,602 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:13,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:13,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:13,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:13,603 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:13,603 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:13,603 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/A is initiating minor compaction (all files) 2024-11-10T04:39:13,603 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/B is initiating minor compaction (all files) 2024-11-10T04:39:13,603 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/B in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:13,603 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/A in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:13,603 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e4be4a1810ac4e28a00e5cac666a1041, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/b9613766772c40ff85b01aae57f97a7f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e97c973643d4425fa27286b20f984101] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=32.9 K 2024-11-10T04:39:13,604 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d146b070f2414e89861454f65ce38c9d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3d3da0d7ff1e4777996ab4cd32b3f4d5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/5c592a7ef59f4ff09c41faab00725294] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=107.5 K 2024-11-10T04:39:13,604 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:13,604 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d146b070f2414e89861454f65ce38c9d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3d3da0d7ff1e4777996ab4cd32b3f4d5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/5c592a7ef59f4ff09c41faab00725294] 2024-11-10T04:39:13,604 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e4be4a1810ac4e28a00e5cac666a1041, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731213551239 2024-11-10T04:39:13,604 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d146b070f2414e89861454f65ce38c9d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731213551237 2024-11-10T04:39:13,604 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b9613766772c40ff85b01aae57f97a7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731213551257 2024-11-10T04:39:13,605 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d3da0d7ff1e4777996ab4cd32b3f4d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731213551257 2024-11-10T04:39:13,605 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e97c973643d4425fa27286b20f984101, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731213553425 2024-11-10T04:39:13,605 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c592a7ef59f4ff09c41faab00725294, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731213553425 2024-11-10T04:39:13,610 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:13,611 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#B#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:13,612 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/2ec7689a7f904f288705ea3db7f93d14 is 50, key is test_row_0/B:col10/1731213553425/Put/seqid=0 2024-11-10T04:39:13,622 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110f255be267cd64fe6807134428e5a12e3_145f65b29b41c5cf34022ad77be79930 store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:13,624 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110f255be267cd64fe6807134428e5a12e3_145f65b29b41c5cf34022ad77be79930, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:13,625 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f255be267cd64fe6807134428e5a12e3_145f65b29b41c5cf34022ad77be79930 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:13,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742334_1510 (size=12104) 2024-11-10T04:39:13,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742335_1511 (size=4469) 2024-11-10T04:39:13,634 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#A#compaction#435 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:13,634 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/02888f868cee407b87a80eb62c100c6f is 175, key is test_row_0/A:col10/1731213553425/Put/seqid=0 2024-11-10T04:39:13,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742336_1512 (size=31058) 2024-11-10T04:39:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:13,685 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:13,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T04:39:13,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100a026a739e124d59b66b6cbed7a560e1_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213553684/Put/seqid=0 2024-11-10T04:39:13,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742337_1513 (size=12154) 2024-11-10T04:39:13,704 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:13,708 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100a026a739e124d59b66b6cbed7a560e1_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100a026a739e124d59b66b6cbed7a560e1_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:13,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213613701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,709 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/abe15a7e00c14ffd8fa28a1abec40340, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:13,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213613701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213613702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/abe15a7e00c14ffd8fa28a1abec40340 is 175, key is test_row_0/A:col10/1731213553684/Put/seqid=0 2024-11-10T04:39:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742338_1514 (size=30955) 2024-11-10T04:39:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213613709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,718 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/abe15a7e00c14ffd8fa28a1abec40340 2024-11-10T04:39:13,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/c09cb8bee781447b947795cda235a8ce is 50, key is test_row_0/B:col10/1731213553684/Put/seqid=0 2024-11-10T04:39:13,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742339_1515 (size=12001) 2024-11-10T04:39:13,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/c09cb8bee781447b947795cda235a8ce 2024-11-10T04:39:13,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/0303f26318f4461ca129d88d6169e09f is 50, key is test_row_0/C:col10/1731213553684/Put/seqid=0 2024-11-10T04:39:13,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742340_1516 (size=12001) 2024-11-10T04:39:13,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213613811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213613821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213613821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213613821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213614021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,030 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/2ec7689a7f904f288705ea3db7f93d14 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/2ec7689a7f904f288705ea3db7f93d14 2024-11-10T04:39:14,034 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/B of 145f65b29b41c5cf34022ad77be79930 into 2ec7689a7f904f288705ea3db7f93d14(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:14,034 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:14,034 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/B, priority=13, startTime=1731213553602; duration=0sec 2024-11-10T04:39:14,034 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:14,034 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:B 2024-11-10T04:39:14,034 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:14,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213614031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213614031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213614031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,035 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:14,035 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/C is initiating minor compaction (all files) 2024-11-10T04:39:14,035 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/C in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:14,036 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/e8bca83d804e41eb99668c2263b73880, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2a4d5ff345ec4be9a42243d158c1cc43, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ed0f09b5df654078a11453bb6364dfc4] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=32.9 K 2024-11-10T04:39:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,036 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e8bca83d804e41eb99668c2263b73880, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731213551239 2024-11-10T04:39:14,036 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a4d5ff345ec4be9a42243d158c1cc43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731213551257 2024-11-10T04:39:14,036 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ed0f09b5df654078a11453bb6364dfc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731213553425 2024-11-10T04:39:14,046 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/02888f868cee407b87a80eb62c100c6f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/02888f868cee407b87a80eb62c100c6f 2024-11-10T04:39:14,047 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:14,048 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/eef2c64485ad4563bc7e3f7debb111cd is 50, key is test_row_0/C:col10/1731213553425/Put/seqid=0 2024-11-10T04:39:14,050 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/A of 145f65b29b41c5cf34022ad77be79930 into 02888f868cee407b87a80eb62c100c6f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:14,050 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:14,050 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/A, priority=13, startTime=1731213553602; duration=0sec 2024-11-10T04:39:14,050 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:14,050 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:A 2024-11-10T04:39:14,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742341_1517 (size=12104) 2024-11-10T04:39:14,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/0303f26318f4461ca129d88d6169e09f 2024-11-10T04:39:14,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/abe15a7e00c14ffd8fa28a1abec40340 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/abe15a7e00c14ffd8fa28a1abec40340 2024-11-10T04:39:14,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/abe15a7e00c14ffd8fa28a1abec40340, entries=150, sequenceid=80, filesize=30.2 K 2024-11-10T04:39:14,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/c09cb8bee781447b947795cda235a8ce as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/c09cb8bee781447b947795cda235a8ce 2024-11-10T04:39:14,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/c09cb8bee781447b947795cda235a8ce, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T04:39:14,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/0303f26318f4461ca129d88d6169e09f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/0303f26318f4461ca129d88d6169e09f 2024-11-10T04:39:14,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/0303f26318f4461ca129d88d6169e09f, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T04:39:14,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 145f65b29b41c5cf34022ad77be79930 in 480ms, sequenceid=80, compaction requested=false 2024-11-10T04:39:14,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:14,334 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:14,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T04:39:14,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111081d82ac200f64cc3ab6774736be83d72_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213553700/Put/seqid=0 2024-11-10T04:39:14,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742342_1518 (size=12154) 2024-11-10T04:39:14,344 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:14,349 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111081d82ac200f64cc3ab6774736be83d72_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111081d82ac200f64cc3ab6774736be83d72_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:14,350 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/029ecf7b91c14fdaab7a93406638b25b, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:14,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/029ecf7b91c14fdaab7a93406638b25b is 175, key is test_row_0/A:col10/1731213553700/Put/seqid=0 2024-11-10T04:39:14,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742343_1519 (size=30955) 2024-11-10T04:39:14,358 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/029ecf7b91c14fdaab7a93406638b25b 2024-11-10T04:39:14,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213614368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213614370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213614371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213614372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/38377ad501ca41dd9b2fe47fd30f7673 is 50, key is test_row_0/B:col10/1731213553700/Put/seqid=0 2024-11-10T04:39:14,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742344_1520 (size=12001) 2024-11-10T04:39:14,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/38377ad501ca41dd9b2fe47fd30f7673 2024-11-10T04:39:14,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/6eae7b6697d3440689032bfbfac410ae is 50, key is test_row_0/C:col10/1731213553700/Put/seqid=0 2024-11-10T04:39:14,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742345_1521 (size=12001) 2024-11-10T04:39:14,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/6eae7b6697d3440689032bfbfac410ae 2024-11-10T04:39:14,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/029ecf7b91c14fdaab7a93406638b25b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/029ecf7b91c14fdaab7a93406638b25b 2024-11-10T04:39:14,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/029ecf7b91c14fdaab7a93406638b25b, entries=150, sequenceid=95, filesize=30.2 K 2024-11-10T04:39:14,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/38377ad501ca41dd9b2fe47fd30f7673 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/38377ad501ca41dd9b2fe47fd30f7673 2024-11-10T04:39:14,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/38377ad501ca41dd9b2fe47fd30f7673, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T04:39:14,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/6eae7b6697d3440689032bfbfac410ae as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6eae7b6697d3440689032bfbfac410ae 2024-11-10T04:39:14,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6eae7b6697d3440689032bfbfac410ae, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T04:39:14,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 145f65b29b41c5cf34022ad77be79930 in 76ms, sequenceid=95, compaction requested=true 2024-11-10T04:39:14,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:14,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:A, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:14,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:14,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:B, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:14,410 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:14,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:14,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:14,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T04:39:14,411 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:14,411 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/A is initiating minor compaction (all files) 2024-11-10T04:39:14,411 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/A in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:14,411 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/02888f868cee407b87a80eb62c100c6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/abe15a7e00c14ffd8fa28a1abec40340, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/029ecf7b91c14fdaab7a93406638b25b] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=90.8 K 2024-11-10T04:39:14,411 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:14,411 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/02888f868cee407b87a80eb62c100c6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/abe15a7e00c14ffd8fa28a1abec40340, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/029ecf7b91c14fdaab7a93406638b25b] 2024-11-10T04:39:14,411 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02888f868cee407b87a80eb62c100c6f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731213553425 2024-11-10T04:39:14,412 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting abe15a7e00c14ffd8fa28a1abec40340, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731213553555 2024-11-10T04:39:14,412 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 029ecf7b91c14fdaab7a93406638b25b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731213553700 2024-11-10T04:39:14,418 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:14,420 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d8388b83e21f4a45bd55215211d5c347_145f65b29b41c5cf34022ad77be79930 store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:14,422 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d8388b83e21f4a45bd55215211d5c347_145f65b29b41c5cf34022ad77be79930, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:14,423 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d8388b83e21f4a45bd55215211d5c347_145f65b29b41c5cf34022ad77be79930 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:14,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742346_1522 (size=4469) 2024-11-10T04:39:14,459 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/eef2c64485ad4563bc7e3f7debb111cd as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/eef2c64485ad4563bc7e3f7debb111cd 2024-11-10T04:39:14,463 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/C of 145f65b29b41c5cf34022ad77be79930 into eef2c64485ad4563bc7e3f7debb111cd(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:14,463 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:14,463 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/C, priority=13, startTime=1731213553603; duration=0sec 2024-11-10T04:39:14,463 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T04:39:14,464 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:14,464 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:14,464 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:14,464 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:14,464 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/B is initiating minor compaction (all files) 2024-11-10T04:39:14,464 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/B in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:14,464 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/2ec7689a7f904f288705ea3db7f93d14, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/c09cb8bee781447b947795cda235a8ce, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/38377ad501ca41dd9b2fe47fd30f7673] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=35.3 K 2024-11-10T04:39:14,465 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec7689a7f904f288705ea3db7f93d14, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731213553425 2024-11-10T04:39:14,465 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c09cb8bee781447b947795cda235a8ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731213553555 2024-11-10T04:39:14,465 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 38377ad501ca41dd9b2fe47fd30f7673, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731213553700 2024-11-10T04:39:14,470 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#B#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:14,471 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/85770e356ff44a0985518e6cdc224565 is 50, key is test_row_0/B:col10/1731213553700/Put/seqid=0 2024-11-10T04:39:14,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742347_1523 (size=12207) 2024-11-10T04:39:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:14,484 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:14,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T04:39:14,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d541fcd4b8064c0f9c88f30ddb90632e_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213554369/Put/seqid=0 2024-11-10T04:39:14,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742348_1524 (size=14594) 2024-11-10T04:39:14,494 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:14,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d541fcd4b8064c0f9c88f30ddb90632e_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d541fcd4b8064c0f9c88f30ddb90632e_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:14,498 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/de54ec33b470420fadb5d5df54888506, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:14,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213614493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/de54ec33b470420fadb5d5df54888506 is 175, key is test_row_0/A:col10/1731213554369/Put/seqid=0 2024-11-10T04:39:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742349_1525 (size=39549) 2024-11-10T04:39:14,501 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/de54ec33b470420fadb5d5df54888506 2024-11-10T04:39:14,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213614498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213614498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213614499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/33842d9d53ab470f87bac1a299b70d74 is 50, key is test_row_0/B:col10/1731213554369/Put/seqid=0 2024-11-10T04:39:14,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742350_1526 (size=12001) 2024-11-10T04:39:14,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213614602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213614611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213614611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213614611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213614811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213614821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213614821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213614821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:14,828 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#A#compaction#444 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:14,829 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/2b9b51f3cf0c4c1f80cbf3cb5dafc74e is 175, key is test_row_0/A:col10/1731213553700/Put/seqid=0 2024-11-10T04:39:14,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742351_1527 (size=31161) 2024-11-10T04:39:14,879 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/85770e356ff44a0985518e6cdc224565 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/85770e356ff44a0985518e6cdc224565 2024-11-10T04:39:14,883 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/B of 145f65b29b41c5cf34022ad77be79930 into 85770e356ff44a0985518e6cdc224565(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:14,883 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:14,883 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/B, priority=13, startTime=1731213554410; duration=0sec 2024-11-10T04:39:14,883 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:14,883 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:B 2024-11-10T04:39:14,883 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:14,884 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:14,884 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/C is initiating minor compaction (all files) 2024-11-10T04:39:14,884 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/C in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:14,884 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/eef2c64485ad4563bc7e3f7debb111cd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/0303f26318f4461ca129d88d6169e09f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6eae7b6697d3440689032bfbfac410ae] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=35.3 K 2024-11-10T04:39:14,884 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting eef2c64485ad4563bc7e3f7debb111cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731213553425 2024-11-10T04:39:14,885 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0303f26318f4461ca129d88d6169e09f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731213553555 2024-11-10T04:39:14,885 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eae7b6697d3440689032bfbfac410ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731213553700 2024-11-10T04:39:14,890 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:14,890 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/59a52f8315eb448b981ca7a832dbe287 is 50, key is test_row_0/C:col10/1731213553700/Put/seqid=0 2024-11-10T04:39:14,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742352_1528 (size=12207) 2024-11-10T04:39:14,903 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/59a52f8315eb448b981ca7a832dbe287 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/59a52f8315eb448b981ca7a832dbe287 2024-11-10T04:39:14,907 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/C of 145f65b29b41c5cf34022ad77be79930 into 59a52f8315eb448b981ca7a832dbe287(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:14,907 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:14,907 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/C, priority=13, startTime=1731213554410; duration=0sec 2024-11-10T04:39:14,907 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:14,907 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:14,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/33842d9d53ab470f87bac1a299b70d74 2024-11-10T04:39:14,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/7894a5e074694c1881f4f9a5d7f7a437 is 50, key is test_row_0/C:col10/1731213554369/Put/seqid=0 2024-11-10T04:39:14,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742353_1529 (size=12001) 2024-11-10T04:39:15,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213615121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213615131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213615132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213615132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,236 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/2b9b51f3cf0c4c1f80cbf3cb5dafc74e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/2b9b51f3cf0c4c1f80cbf3cb5dafc74e 2024-11-10T04:39:15,240 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/A of 145f65b29b41c5cf34022ad77be79930 into 2b9b51f3cf0c4c1f80cbf3cb5dafc74e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:15,240 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:15,240 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/A, priority=13, startTime=1731213554410; duration=0sec 2024-11-10T04:39:15,240 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:15,240 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:A 2024-11-10T04:39:15,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/7894a5e074694c1881f4f9a5d7f7a437 2024-11-10T04:39:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/de54ec33b470420fadb5d5df54888506 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/de54ec33b470420fadb5d5df54888506 2024-11-10T04:39:15,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/de54ec33b470420fadb5d5df54888506, entries=200, sequenceid=120, filesize=38.6 K 2024-11-10T04:39:15,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/33842d9d53ab470f87bac1a299b70d74 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/33842d9d53ab470f87bac1a299b70d74 2024-11-10T04:39:15,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/33842d9d53ab470f87bac1a299b70d74, entries=150, sequenceid=120, filesize=11.7 K 2024-11-10T04:39:15,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/7894a5e074694c1881f4f9a5d7f7a437 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/7894a5e074694c1881f4f9a5d7f7a437 2024-11-10T04:39:15,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/7894a5e074694c1881f4f9a5d7f7a437, entries=150, sequenceid=120, filesize=11.7 K 2024-11-10T04:39:15,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 145f65b29b41c5cf34022ad77be79930 in 857ms, sequenceid=120, compaction requested=false 2024-11-10T04:39:15,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T04:39:15,391 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:15,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:39:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-10T04:39:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T04:39:15,393 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:39:15,394 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:39:15,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:39:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:15,443 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:15,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T04:39:15,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108c302c19da724bb791f25a13cbdecbfb_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213554497/Put/seqid=0 2024-11-10T04:39:15,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742354_1530 (size=12304) 2024-11-10T04:39:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T04:39:15,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213615528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:15,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:15,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213615631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213615641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213615641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213615641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213615641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:15,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:15,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:15,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:15,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:15,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T04:39:15,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:15,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:15,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:15,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:15,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:15,853 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:15,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213615851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:15,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:15,857 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108c302c19da724bb791f25a13cbdecbfb_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108c302c19da724bb791f25a13cbdecbfb_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:15,858 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/45bbaa5ea95749f6a8b6e44adb7797ab, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:15,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/45bbaa5ea95749f6a8b6e44adb7797ab is 175, key is test_row_0/A:col10/1731213554497/Put/seqid=0 2024-11-10T04:39:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742355_1531 (size=31105) 2024-11-10T04:39:16,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T04:39:16,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213616161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:16,263 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/45bbaa5ea95749f6a8b6e44adb7797ab 2024-11-10T04:39:16,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/def86dbfe1e94d05a431b335d1e492f9 is 50, key is test_row_0/B:col10/1731213554497/Put/seqid=0 2024-11-10T04:39:16,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742356_1532 (size=12151) 2024-11-10T04:39:16,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T04:39:16,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:16,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213616651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:16,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213616651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:16,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213616651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:16,653 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:16,654 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_0' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:16,654 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_0' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:16,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213616661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:16,664 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_0' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:16,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:16,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213616670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:16,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/def86dbfe1e94d05a431b335d1e492f9 2024-11-10T04:39:16,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/262512bfe883410ea7fae432e7c3f851 is 50, key is test_row_0/C:col10/1731213554497/Put/seqid=0 2024-11-10T04:39:16,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742357_1533 (size=12151) 2024-11-10T04:39:16,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:16,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:17,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:17,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:17,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:17,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:17,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:17,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:17,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/262512bfe883410ea7fae432e7c3f851 2024-11-10T04:39:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/45bbaa5ea95749f6a8b6e44adb7797ab as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/45bbaa5ea95749f6a8b6e44adb7797ab 2024-11-10T04:39:17,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/45bbaa5ea95749f6a8b6e44adb7797ab, entries=150, sequenceid=137, filesize=30.4 K 2024-11-10T04:39:17,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/def86dbfe1e94d05a431b335d1e492f9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/def86dbfe1e94d05a431b335d1e492f9 2024-11-10T04:39:17,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/def86dbfe1e94d05a431b335d1e492f9, entries=150, sequenceid=137, filesize=11.9 K 2024-11-10T04:39:17,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/262512bfe883410ea7fae432e7c3f851 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/262512bfe883410ea7fae432e7c3f851 2024-11-10T04:39:17,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/262512bfe883410ea7fae432e7c3f851, entries=150, sequenceid=137, filesize=11.9 K 2024-11-10T04:39:17,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 145f65b29b41c5cf34022ad77be79930 in 1661ms, sequenceid=137, compaction requested=true 2024-11-10T04:39:17,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:17,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:17,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:17,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:17,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:17,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:17,105 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:17,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:17,105 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:17,105 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:17,105 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:17,106 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/A is initiating minor compaction (all files) 2024-11-10T04:39:17,106 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/B is initiating minor compaction (all files) 2024-11-10T04:39:17,106 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/B in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:17,106 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/A in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:17,106 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/85770e356ff44a0985518e6cdc224565, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/33842d9d53ab470f87bac1a299b70d74, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/def86dbfe1e94d05a431b335d1e492f9] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=35.5 K 2024-11-10T04:39:17,106 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/2b9b51f3cf0c4c1f80cbf3cb5dafc74e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/de54ec33b470420fadb5d5df54888506, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/45bbaa5ea95749f6a8b6e44adb7797ab] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=99.4 K 2024-11-10T04:39:17,106 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:17,106 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/2b9b51f3cf0c4c1f80cbf3cb5dafc74e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/de54ec33b470420fadb5d5df54888506, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/45bbaa5ea95749f6a8b6e44adb7797ab] 2024-11-10T04:39:17,106 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b9b51f3cf0c4c1f80cbf3cb5dafc74e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731213553700 2024-11-10T04:39:17,106 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 85770e356ff44a0985518e6cdc224565, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731213553700 2024-11-10T04:39:17,106 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 33842d9d53ab470f87bac1a299b70d74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731213554366 2024-11-10T04:39:17,106 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting de54ec33b470420fadb5d5df54888506, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731213554366 2024-11-10T04:39:17,107 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting def86dbfe1e94d05a431b335d1e492f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731213554492 2024-11-10T04:39:17,107 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45bbaa5ea95749f6a8b6e44adb7797ab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731213554492 2024-11-10T04:39:17,112 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:17,112 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#B#compaction#453 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:17,113 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/991d8a6babd8464d86ce161d7a6a71d4 is 50, key is test_row_0/B:col10/1731213554497/Put/seqid=0 2024-11-10T04:39:17,115 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d54bbc18035d44d1a96e9f034f5f8ed1_145f65b29b41c5cf34022ad77be79930 store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:17,116 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d54bbc18035d44d1a96e9f034f5f8ed1_145f65b29b41c5cf34022ad77be79930, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:17,116 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d54bbc18035d44d1a96e9f034f5f8ed1_145f65b29b41c5cf34022ad77be79930 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:17,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742358_1534 (size=12459) 2024-11-10T04:39:17,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742359_1535 (size=4469) 2024-11-10T04:39:17,122 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#A#compaction#454 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:17,122 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/da57cb59c7fa47c597997600192a956e is 175, key is test_row_0/A:col10/1731213554497/Put/seqid=0 2024-11-10T04:39:17,124 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/991d8a6babd8464d86ce161d7a6a71d4 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/991d8a6babd8464d86ce161d7a6a71d4 2024-11-10T04:39:17,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742360_1536 (size=31413) 2024-11-10T04:39:17,129 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/B of 145f65b29b41c5cf34022ad77be79930 into 991d8a6babd8464d86ce161d7a6a71d4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:17,129 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:17,129 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/B, priority=13, startTime=1731213557105; duration=0sec 2024-11-10T04:39:17,129 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:17,129 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:B 2024-11-10T04:39:17,129 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:17,130 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:17,130 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/C is initiating minor compaction (all files) 2024-11-10T04:39:17,130 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/C in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:17,130 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/59a52f8315eb448b981ca7a832dbe287, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/7894a5e074694c1881f4f9a5d7f7a437, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/262512bfe883410ea7fae432e7c3f851] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=35.5 K 2024-11-10T04:39:17,131 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a52f8315eb448b981ca7a832dbe287, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731213553700 2024-11-10T04:39:17,131 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7894a5e074694c1881f4f9a5d7f7a437, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731213554366 2024-11-10T04:39:17,131 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 262512bfe883410ea7fae432e7c3f851, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731213554492 2024-11-10T04:39:17,132 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/da57cb59c7fa47c597997600192a956e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/da57cb59c7fa47c597997600192a956e 2024-11-10T04:39:17,137 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/A of 145f65b29b41c5cf34022ad77be79930 into da57cb59c7fa47c597997600192a956e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:17,137 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:17,137 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/A, priority=13, startTime=1731213557104; duration=0sec 2024-11-10T04:39:17,137 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:17,137 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:A 2024-11-10T04:39:17,141 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#C#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:17,141 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/c6d350d9a73047f69881e8951cd9b686 is 50, key is test_row_0/C:col10/1731213554497/Put/seqid=0 2024-11-10T04:39:17,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742361_1537 (size=12459) 2024-11-10T04:39:17,158 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/c6d350d9a73047f69881e8951cd9b686 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6d350d9a73047f69881e8951cd9b686 2024-11-10T04:39:17,164 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/C of 145f65b29b41c5cf34022ad77be79930 into c6d350d9a73047f69881e8951cd9b686(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:17,164 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:17,164 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/C, priority=13, startTime=1731213557105; duration=0sec 2024-11-10T04:39:17,164 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:17,164 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:17,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T04:39:17,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:17,230 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T04:39:17,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110685bae4d6f144260b93b1cb42f4083fe_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213555503/Put/seqid=0 2024-11-10T04:39:17,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742362_1538 (size=12304) 2024-11-10T04:39:17,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:17,245 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110685bae4d6f144260b93b1cb42f4083fe_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110685bae4d6f144260b93b1cb42f4083fe_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:17,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/9275f8ebf874412cbbae287af6034923, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:17,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/9275f8ebf874412cbbae287af6034923 is 175, key is test_row_0/A:col10/1731213555503/Put/seqid=0 2024-11-10T04:39:17,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742363_1539 (size=31105) 2024-11-10T04:39:17,250 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/9275f8ebf874412cbbae287af6034923 2024-11-10T04:39:17,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/4283d8cd5b6a497a9544eb854d00629c is 50, key is test_row_0/B:col10/1731213555503/Put/seqid=0 2024-11-10T04:39:17,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742364_1540 (size=12151) 2024-11-10T04:39:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T04:39:17,660 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/4283d8cd5b6a497a9544eb854d00629c 2024-11-10T04:39:17,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/d0f2c3682f424c97a53d609ca7fda553 is 50, key is test_row_0/C:col10/1731213555503/Put/seqid=0 2024-11-10T04:39:17,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742365_1541 (size=12151) 2024-11-10T04:39:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:17,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:17,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213617720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:17,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213617831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:17,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:17,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:18,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213618041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:18,071 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/d0f2c3682f424c97a53d609ca7fda553 2024-11-10T04:39:18,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/9275f8ebf874412cbbae287af6034923 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/9275f8ebf874412cbbae287af6034923 2024-11-10T04:39:18,078 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/9275f8ebf874412cbbae287af6034923, entries=150, sequenceid=162, filesize=30.4 K 2024-11-10T04:39:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/4283d8cd5b6a497a9544eb854d00629c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4283d8cd5b6a497a9544eb854d00629c 2024-11-10T04:39:18,082 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4283d8cd5b6a497a9544eb854d00629c, entries=150, sequenceid=162, filesize=11.9 K 2024-11-10T04:39:18,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/d0f2c3682f424c97a53d609ca7fda553 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0f2c3682f424c97a53d609ca7fda553 2024-11-10T04:39:18,087 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0f2c3682f424c97a53d609ca7fda553, entries=150, sequenceid=162, filesize=11.9 K 2024-11-10T04:39:18,087 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 145f65b29b41c5cf34022ad77be79930 in 857ms, sequenceid=162, compaction requested=false 2024-11-10T04:39:18,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:18,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:18,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-10T04:39:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-10T04:39:18,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-10T04:39:18,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6950 sec 2024-11-10T04:39:18,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.7000 sec 2024-11-10T04:39:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:18,353 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:18,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T04:39:18,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111026f57b0b02e747d3824aab589f634a75_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213557715/Put/seqid=0 2024-11-10T04:39:18,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742366_1542 (size=12304) 2024-11-10T04:39:18,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213618439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:18,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213618551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:18,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213618671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:18,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213618671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:18,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213618671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:18,674 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_0' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:18,675 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:18,675 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_0' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213618681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:18,685 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_0' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:18,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213618761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:18,765 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:18,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111026f57b0b02e747d3824aab589f634a75_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111026f57b0b02e747d3824aab589f634a75_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:18,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/01994e9576c44ee9b612478befe31063, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:18,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/01994e9576c44ee9b612478befe31063 is 175, key is test_row_0/A:col10/1731213557715/Put/seqid=0 2024-11-10T04:39:18,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742367_1543 (size=31105) 2024-11-10T04:39:19,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213619071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:19,173 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/01994e9576c44ee9b612478befe31063 2024-11-10T04:39:19,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/797c6fd27da04409b59ea314229acc8d is 50, key is test_row_0/B:col10/1731213557715/Put/seqid=0 2024-11-10T04:39:19,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742368_1544 (size=12151) 2024-11-10T04:39:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T04:39:19,551 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:19,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:39:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-10T04:39:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T04:39:19,554 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:39:19,554 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:39:19,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:39:19,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/797c6fd27da04409b59ea314229acc8d 2024-11-10T04:39:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213619581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:19,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/dd22068f46da46b6b874d656bec341eb is 50, key is test_row_0/C:col10/1731213557715/Put/seqid=0 2024-11-10T04:39:19,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742369_1545 (size=12151) 2024-11-10T04:39:19,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/dd22068f46da46b6b874d656bec341eb 2024-11-10T04:39:19,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/01994e9576c44ee9b612478befe31063 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/01994e9576c44ee9b612478befe31063 2024-11-10T04:39:19,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/01994e9576c44ee9b612478befe31063, entries=150, sequenceid=177, filesize=30.4 K 2024-11-10T04:39:19,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/797c6fd27da04409b59ea314229acc8d as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/797c6fd27da04409b59ea314229acc8d 2024-11-10T04:39:19,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/797c6fd27da04409b59ea314229acc8d, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T04:39:19,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/dd22068f46da46b6b874d656bec341eb as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/dd22068f46da46b6b874d656bec341eb 2024-11-10T04:39:19,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/dd22068f46da46b6b874d656bec341eb, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T04:39:19,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 145f65b29b41c5cf34022ad77be79930 in 1254ms, sequenceid=177, compaction requested=true 2024-11-10T04:39:19,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:19,608 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:19,608 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:19,608 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:19,608 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:19,609 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/A is initiating minor compaction (all files) 2024-11-10T04:39:19,609 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/B is initiating minor compaction (all files) 2024-11-10T04:39:19,609 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/A in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:19,609 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/B in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:19,609 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/da57cb59c7fa47c597997600192a956e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/9275f8ebf874412cbbae287af6034923, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/01994e9576c44ee9b612478befe31063] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=91.4 K 2024-11-10T04:39:19,609 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/991d8a6babd8464d86ce161d7a6a71d4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4283d8cd5b6a497a9544eb854d00629c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/797c6fd27da04409b59ea314229acc8d] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=35.9 K 2024-11-10T04:39:19,609 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:19,609 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/da57cb59c7fa47c597997600192a956e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/9275f8ebf874412cbbae287af6034923, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/01994e9576c44ee9b612478befe31063] 2024-11-10T04:39:19,609 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 991d8a6babd8464d86ce161d7a6a71d4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731213554492 2024-11-10T04:39:19,609 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting da57cb59c7fa47c597997600192a956e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731213554492 2024-11-10T04:39:19,609 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9275f8ebf874412cbbae287af6034923, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731213555498 2024-11-10T04:39:19,609 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4283d8cd5b6a497a9544eb854d00629c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731213555498 2024-11-10T04:39:19,610 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01994e9576c44ee9b612478befe31063, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731213557715 2024-11-10T04:39:19,610 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 797c6fd27da04409b59ea314229acc8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731213557715 2024-11-10T04:39:19,616 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#B#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:19,617 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/84c7fde6e6554eadb0e671a6d298dc92 is 50, key is test_row_0/B:col10/1731213557715/Put/seqid=0 2024-11-10T04:39:19,618 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:19,620 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411101e96e654affb4c248f0ecf3e40fd47db_145f65b29b41c5cf34022ad77be79930 store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:19,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742370_1546 (size=12561) 2024-11-10T04:39:19,622 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411101e96e654affb4c248f0ecf3e40fd47db_145f65b29b41c5cf34022ad77be79930, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:19,622 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e96e654affb4c248f0ecf3e40fd47db_145f65b29b41c5cf34022ad77be79930 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:19,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742371_1547 (size=4469) 2024-11-10T04:39:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T04:39:19,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T04:39:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:19,707 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T04:39:19,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e8692f5089e40eda4b734ae1d8230fd_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213558438/Put/seqid=0 2024-11-10T04:39:19,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742372_1548 (size=12304) 2024-11-10T04:39:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T04:39:20,028 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/84c7fde6e6554eadb0e671a6d298dc92 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/84c7fde6e6554eadb0e671a6d298dc92 2024-11-10T04:39:20,032 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#A#compaction#463 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:20,032 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/B of 145f65b29b41c5cf34022ad77be79930 into 84c7fde6e6554eadb0e671a6d298dc92(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:20,032 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:20,032 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/B, priority=13, startTime=1731213559608; duration=0sec 2024-11-10T04:39:20,032 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:20,032 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:B 2024-11-10T04:39:20,032 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:20,033 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/8bfad6d2b5d54b069d5a7799d039f419 is 175, key is test_row_0/A:col10/1731213557715/Put/seqid=0 2024-11-10T04:39:20,033 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:20,033 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/C is initiating minor compaction (all files) 2024-11-10T04:39:20,033 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/C in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:20,033 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6d350d9a73047f69881e8951cd9b686, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0f2c3682f424c97a53d609ca7fda553, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/dd22068f46da46b6b874d656bec341eb] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=35.9 K 2024-11-10T04:39:20,034 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c6d350d9a73047f69881e8951cd9b686, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731213554492 2024-11-10T04:39:20,034 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f2c3682f424c97a53d609ca7fda553, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731213555498 2024-11-10T04:39:20,035 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting dd22068f46da46b6b874d656bec341eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731213557715 2024-11-10T04:39:20,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742373_1549 (size=31515) 2024-11-10T04:39:20,040 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:20,041 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/d0e939a887a84570bc37a2e2aa439b07 is 50, key is test_row_0/C:col10/1731213557715/Put/seqid=0 2024-11-10T04:39:20,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742374_1550 (size=12561) 2024-11-10T04:39:20,046 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/d0e939a887a84570bc37a2e2aa439b07 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0e939a887a84570bc37a2e2aa439b07 2024-11-10T04:39:20,050 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/C of 145f65b29b41c5cf34022ad77be79930 into d0e939a887a84570bc37a2e2aa439b07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:20,050 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:20,050 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/C, priority=13, startTime=1731213559608; duration=0sec 2024-11-10T04:39:20,050 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:20,050 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:20,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:20,122 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101e8692f5089e40eda4b734ae1d8230fd_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e8692f5089e40eda4b734ae1d8230fd_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:20,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/b68ce68cfa5c4d77b348451acd3e3e57, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:20,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/b68ce68cfa5c4d77b348451acd3e3e57 is 175, key is test_row_0/A:col10/1731213558438/Put/seqid=0 2024-11-10T04:39:20,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742375_1551 (size=31105) 2024-11-10T04:39:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T04:39:20,441 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/8bfad6d2b5d54b069d5a7799d039f419 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8bfad6d2b5d54b069d5a7799d039f419 2024-11-10T04:39:20,445 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/A of 145f65b29b41c5cf34022ad77be79930 into 8bfad6d2b5d54b069d5a7799d039f419(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:20,445 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:20,445 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/A, priority=13, startTime=1731213559607; duration=0sec 2024-11-10T04:39:20,445 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:20,445 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:A 2024-11-10T04:39:20,534 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/b68ce68cfa5c4d77b348451acd3e3e57 2024-11-10T04:39:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/ff6bd64ca21a4d838cb7141ff36290e5 is 50, key is test_row_0/B:col10/1731213558438/Put/seqid=0 2024-11-10T04:39:20,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742376_1552 (size=12151) 2024-11-10T04:39:20,545 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/ff6bd64ca21a4d838cb7141ff36290e5 2024-11-10T04:39:20,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/6a57fc76e80b41c08047dcc64f479bc1 is 50, key is test_row_0/C:col10/1731213558438/Put/seqid=0 2024-11-10T04:39:20,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742377_1553 (size=12151) 2024-11-10T04:39:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:20,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213620640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T04:39:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213620751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:20,955 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/6a57fc76e80b41c08047dcc64f479bc1 2024-11-10T04:39:20,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/b68ce68cfa5c4d77b348451acd3e3e57 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b68ce68cfa5c4d77b348451acd3e3e57 2024-11-10T04:39:20,962 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b68ce68cfa5c4d77b348451acd3e3e57, entries=150, sequenceid=198, filesize=30.4 K 2024-11-10T04:39:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/ff6bd64ca21a4d838cb7141ff36290e5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/ff6bd64ca21a4d838cb7141ff36290e5 2024-11-10T04:39:20,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213620961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:20,966 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/ff6bd64ca21a4d838cb7141ff36290e5, entries=150, sequenceid=198, filesize=11.9 K 2024-11-10T04:39:20,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/6a57fc76e80b41c08047dcc64f479bc1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6a57fc76e80b41c08047dcc64f479bc1 2024-11-10T04:39:20,970 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6a57fc76e80b41c08047dcc64f479bc1, entries=150, sequenceid=198, filesize=11.9 K 2024-11-10T04:39:20,971 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 145f65b29b41c5cf34022ad77be79930 in 1264ms, sequenceid=198, compaction requested=false 2024-11-10T04:39:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:20,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-10T04:39:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-10T04:39:20,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-10T04:39:20,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-10T04:39:20,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4230 sec 2024-11-10T04:39:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:21,273 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:21,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T04:39:21,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100880db167b744e579057e53c2eb016fd_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213561272/Put/seqid=0 2024-11-10T04:39:21,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742378_1554 (size=14794) 2024-11-10T04:39:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213621357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:21,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213621471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:21,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213621681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:21,684 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:21,688 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100880db167b744e579057e53c2eb016fd_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100880db167b744e579057e53c2eb016fd_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:21,688 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/bf6c7009200544349ad18f826e9b5cca, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:21,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/bf6c7009200544349ad18f826e9b5cca is 175, key is test_row_0/A:col10/1731213561272/Put/seqid=0 2024-11-10T04:39:21,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742379_1555 (size=39749) 2024-11-10T04:39:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T04:39:21,701 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:21,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:39:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-10T04:39:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T04:39:21,703 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:39:21,704 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:39:21,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:39:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T04:39:21,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T04:39:21,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:21,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:21,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:21,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:21,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:21,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213621991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T04:39:22,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:22,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T04:39:22,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/bf6c7009200544349ad18f826e9b5cca 2024-11-10T04:39:22,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/64ab41c6ccf04edd9c114fc6bb9cf7ff is 50, key is test_row_0/B:col10/1731213561272/Put/seqid=0 2024-11-10T04:39:22,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742380_1556 (size=12151) 2024-11-10T04:39:22,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/64ab41c6ccf04edd9c114fc6bb9cf7ff 2024-11-10T04:39:22,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/79923df1ac814d728a78a4cc0a4583de is 50, key is test_row_0/C:col10/1731213561272/Put/seqid=0 2024-11-10T04:39:22,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742381_1557 (size=12151) 2024-11-10T04:39:22,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T04:39:22,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:22,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T04:39:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T04:39:22,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T04:39:22,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:22,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:22,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213622501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:22,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/79923df1ac814d728a78a4cc0a4583de 2024-11-10T04:39:22,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/bf6c7009200544349ad18f826e9b5cca as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/bf6c7009200544349ad18f826e9b5cca 2024-11-10T04:39:22,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/bf6c7009200544349ad18f826e9b5cca, entries=200, sequenceid=217, filesize=38.8 K 2024-11-10T04:39:22,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/64ab41c6ccf04edd9c114fc6bb9cf7ff as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/64ab41c6ccf04edd9c114fc6bb9cf7ff 2024-11-10T04:39:22,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/64ab41c6ccf04edd9c114fc6bb9cf7ff, entries=150, sequenceid=217, filesize=11.9 K 2024-11-10T04:39:22,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/79923df1ac814d728a78a4cc0a4583de as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/79923df1ac814d728a78a4cc0a4583de 2024-11-10T04:39:22,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/79923df1ac814d728a78a4cc0a4583de, entries=150, sequenceid=217, filesize=11.9 K 2024-11-10T04:39:22,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 145f65b29b41c5cf34022ad77be79930 in 1256ms, sequenceid=217, compaction requested=true 2024-11-10T04:39:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:22,529 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:22,530 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:22,530 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:22,530 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:22,530 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/A is initiating minor compaction (all files) 2024-11-10T04:39:22,530 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/B is initiating minor compaction (all files) 2024-11-10T04:39:22,530 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/B in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,530 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/A in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,531 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/84c7fde6e6554eadb0e671a6d298dc92, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/ff6bd64ca21a4d838cb7141ff36290e5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/64ab41c6ccf04edd9c114fc6bb9cf7ff] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.0 K 2024-11-10T04:39:22,531 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8bfad6d2b5d54b069d5a7799d039f419, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b68ce68cfa5c4d77b348451acd3e3e57, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/bf6c7009200544349ad18f826e9b5cca] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=100.0 K 2024-11-10T04:39:22,531 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,531 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8bfad6d2b5d54b069d5a7799d039f419, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b68ce68cfa5c4d77b348451acd3e3e57, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/bf6c7009200544349ad18f826e9b5cca] 2024-11-10T04:39:22,531 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bfad6d2b5d54b069d5a7799d039f419, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731213557715 2024-11-10T04:39:22,531 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 84c7fde6e6554eadb0e671a6d298dc92, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731213557715 2024-11-10T04:39:22,531 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ff6bd64ca21a4d838cb7141ff36290e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731213558419 2024-11-10T04:39:22,531 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b68ce68cfa5c4d77b348451acd3e3e57, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731213558419 2024-11-10T04:39:22,532 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf6c7009200544349ad18f826e9b5cca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731213560628 2024-11-10T04:39:22,532 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ab41c6ccf04edd9c114fc6bb9cf7ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731213560628 2024-11-10T04:39:22,536 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:22,538 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111000837dc4342c4185b7c30551275ca9a0_145f65b29b41c5cf34022ad77be79930 store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:22,540 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111000837dc4342c4185b7c30551275ca9a0_145f65b29b41c5cf34022ad77be79930, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:22,541 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111000837dc4342c4185b7c30551275ca9a0_145f65b29b41c5cf34022ad77be79930 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:22,541 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#B#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:22,542 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/6dee58aea49b4bbf96e47d77623c1709 is 50, key is test_row_0/B:col10/1731213561272/Put/seqid=0 2024-11-10T04:39:22,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742383_1559 (size=12663) 2024-11-10T04:39:22,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742382_1558 (size=4469) 2024-11-10T04:39:22,561 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/6dee58aea49b4bbf96e47d77623c1709 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/6dee58aea49b4bbf96e47d77623c1709 2024-11-10T04:39:22,565 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/B of 145f65b29b41c5cf34022ad77be79930 into 6dee58aea49b4bbf96e47d77623c1709(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:22,565 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:22,565 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/B, priority=13, startTime=1731213562529; duration=0sec 2024-11-10T04:39:22,565 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:22,565 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:B 2024-11-10T04:39:22,565 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:22,566 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:22,566 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/C is initiating minor compaction (all files) 2024-11-10T04:39:22,566 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/C in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,566 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0e939a887a84570bc37a2e2aa439b07, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6a57fc76e80b41c08047dcc64f479bc1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/79923df1ac814d728a78a4cc0a4583de] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.0 K 2024-11-10T04:39:22,567 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e939a887a84570bc37a2e2aa439b07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731213557715 2024-11-10T04:39:22,567 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a57fc76e80b41c08047dcc64f479bc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731213558419 2024-11-10T04:39:22,567 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 79923df1ac814d728a78a4cc0a4583de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731213560628 2024-11-10T04:39:22,572 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#C#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:22,573 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/c6ee23b700344d9884c6f3dbd5d0bbe3 is 50, key is test_row_0/C:col10/1731213561272/Put/seqid=0 2024-11-10T04:39:22,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742384_1560 (size=12663) 2024-11-10T04:39:22,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T04:39:22,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:22,622 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T04:39:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101a0531df228f456eb3c8f5796a87350f_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213561340/Put/seqid=0 2024-11-10T04:39:22,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742385_1561 (size=12304) 2024-11-10T04:39:22,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:22,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213622725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213622726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213622726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213622727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213622831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T04:39:22,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213622841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213622841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213622842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:22,959 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#A#compaction#471 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:22,959 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/ce6826bd76be4580bb3389135faab171 is 175, key is test_row_0/A:col10/1731213561272/Put/seqid=0 2024-11-10T04:39:22,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742386_1562 (size=31617) 2024-11-10T04:39:22,980 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/c6ee23b700344d9884c6f3dbd5d0bbe3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6ee23b700344d9884c6f3dbd5d0bbe3 2024-11-10T04:39:22,984 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/C of 145f65b29b41c5cf34022ad77be79930 into c6ee23b700344d9884c6f3dbd5d0bbe3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:22,984 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:22,984 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/C, priority=13, startTime=1731213562529; duration=0sec 2024-11-10T04:39:22,984 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:22,984 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:23,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:23,035 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101a0531df228f456eb3c8f5796a87350f_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101a0531df228f456eb3c8f5796a87350f_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:23,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/c9cebce0e84a4b88ad5d188135883e68, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/c9cebce0e84a4b88ad5d188135883e68 is 175, key is test_row_0/A:col10/1731213561340/Put/seqid=0 2024-11-10T04:39:23,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742387_1563 (size=31105) 2024-11-10T04:39:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213623041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213623051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213623051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213623051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213623351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213623361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213623361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213623361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,368 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/ce6826bd76be4580bb3389135faab171 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ce6826bd76be4580bb3389135faab171 2024-11-10T04:39:23,372 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/A of 145f65b29b41c5cf34022ad77be79930 into ce6826bd76be4580bb3389135faab171(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:23,372 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:23,372 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/A, priority=13, startTime=1731213562529; duration=0sec 2024-11-10T04:39:23,372 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:23,372 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:A 2024-11-10T04:39:23,439 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/c9cebce0e84a4b88ad5d188135883e68 2024-11-10T04:39:23,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/05ca1f13459e4345bdd3b7f214201588 is 50, key is test_row_0/B:col10/1731213561340/Put/seqid=0 2024-11-10T04:39:23,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742388_1564 (size=12151) 2024-11-10T04:39:23,454 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/05ca1f13459e4345bdd3b7f214201588 2024-11-10T04:39:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/9317f1f9bf8843b4b349d28e6d4f561b is 50, key is test_row_0/C:col10/1731213561340/Put/seqid=0 2024-11-10T04:39:23,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742389_1565 (size=12151) 2024-11-10T04:39:23,465 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/9317f1f9bf8843b4b349d28e6d4f561b 2024-11-10T04:39:23,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/c9cebce0e84a4b88ad5d188135883e68 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c9cebce0e84a4b88ad5d188135883e68 2024-11-10T04:39:23,471 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c9cebce0e84a4b88ad5d188135883e68, entries=150, sequenceid=238, filesize=30.4 K 2024-11-10T04:39:23,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/05ca1f13459e4345bdd3b7f214201588 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/05ca1f13459e4345bdd3b7f214201588 2024-11-10T04:39:23,475 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/05ca1f13459e4345bdd3b7f214201588, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T04:39:23,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/9317f1f9bf8843b4b349d28e6d4f561b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/9317f1f9bf8843b4b349d28e6d4f561b 2024-11-10T04:39:23,479 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/9317f1f9bf8843b4b349d28e6d4f561b, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T04:39:23,480 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 145f65b29b41c5cf34022ad77be79930 in 858ms, sequenceid=238, compaction requested=false 2024-11-10T04:39:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-10T04:39:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-10T04:39:23,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-10T04:39:23,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-11-10T04:39:23,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.7820 sec 2024-11-10T04:39:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:23,513 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:23,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T04:39:23,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bc2808d4c7164c96b662862ae381f40c_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213562726/Put/seqid=0 2024-11-10T04:39:23,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742390_1566 (size=14844) 2024-11-10T04:39:23,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213623586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213623701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T04:39:23,852 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:23,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:39:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-10T04:39:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T04:39:23,854 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:39:23,854 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:39:23,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:39:23,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213623861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213623871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213623871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213623871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213623911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:23,923 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:23,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bc2808d4c7164c96b662862ae381f40c_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bc2808d4c7164c96b662862ae381f40c_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:23,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/d200df297fb24f1c8a8c5b4ec1355bc0, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:23,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/d200df297fb24f1c8a8c5b4ec1355bc0 is 175, key is test_row_0/A:col10/1731213562726/Put/seqid=0 2024-11-10T04:39:23,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742391_1567 (size=39799) 2024-11-10T04:39:23,931 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/d200df297fb24f1c8a8c5b4ec1355bc0 2024-11-10T04:39:23,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/4e521da0cb65495fb791a532cff9e244 is 50, key is test_row_0/B:col10/1731213562726/Put/seqid=0 2024-11-10T04:39:23,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742392_1568 (size=12201) 2024-11-10T04:39:23,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/4e521da0cb65495fb791a532cff9e244 2024-11-10T04:39:23,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/77f60c6f7c42458880a1d516b5128ac9 is 50, key is test_row_0/C:col10/1731213562726/Put/seqid=0 2024-11-10T04:39:23,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742393_1569 (size=12201) 2024-11-10T04:39:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T04:39:24,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T04:39:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:24,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T04:39:24,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:24,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:24,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T04:39:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213624221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:24,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T04:39:24,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:24,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:24,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:24,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/77f60c6f7c42458880a1d516b5128ac9 2024-11-10T04:39:24,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/d200df297fb24f1c8a8c5b4ec1355bc0 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d200df297fb24f1c8a8c5b4ec1355bc0 2024-11-10T04:39:24,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d200df297fb24f1c8a8c5b4ec1355bc0, entries=200, sequenceid=257, filesize=38.9 K 2024-11-10T04:39:24,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/4e521da0cb65495fb791a532cff9e244 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4e521da0cb65495fb791a532cff9e244 2024-11-10T04:39:24,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4e521da0cb65495fb791a532cff9e244, entries=150, sequenceid=257, filesize=11.9 K 2024-11-10T04:39:24,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/77f60c6f7c42458880a1d516b5128ac9 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/77f60c6f7c42458880a1d516b5128ac9 2024-11-10T04:39:24,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/77f60c6f7c42458880a1d516b5128ac9, entries=150, sequenceid=257, filesize=11.9 K 2024-11-10T04:39:24,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 145f65b29b41c5cf34022ad77be79930 in 853ms, sequenceid=257, compaction requested=true 2024-11-10T04:39:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:24,367 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:24,367 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:24,367 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:24,368 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/A is initiating minor compaction (all files) 2024-11-10T04:39:24,368 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/A in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,368 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ce6826bd76be4580bb3389135faab171, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c9cebce0e84a4b88ad5d188135883e68, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d200df297fb24f1c8a8c5b4ec1355bc0] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=100.1 K 2024-11-10T04:39:24,368 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,368 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ce6826bd76be4580bb3389135faab171, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c9cebce0e84a4b88ad5d188135883e68, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d200df297fb24f1c8a8c5b4ec1355bc0] 2024-11-10T04:39:24,368 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:24,368 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6826bd76be4580bb3389135faab171, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731213560628 2024-11-10T04:39:24,368 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/B is initiating minor compaction (all files) 2024-11-10T04:39:24,368 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/B in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,368 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/6dee58aea49b4bbf96e47d77623c1709, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/05ca1f13459e4345bdd3b7f214201588, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4e521da0cb65495fb791a532cff9e244] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.1 K 2024-11-10T04:39:24,368 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9cebce0e84a4b88ad5d188135883e68, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731213561340 2024-11-10T04:39:24,369 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dee58aea49b4bbf96e47d77623c1709, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731213560628 2024-11-10T04:39:24,369 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d200df297fb24f1c8a8c5b4ec1355bc0, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731213562723 2024-11-10T04:39:24,369 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ca1f13459e4345bdd3b7f214201588, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731213561340 2024-11-10T04:39:24,369 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e521da0cb65495fb791a532cff9e244, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731213562723 2024-11-10T04:39:24,373 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:24,375 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411102990b1978be84474805c1825d06242df_145f65b29b41c5cf34022ad77be79930 store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:24,375 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#B#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:24,376 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/fff0d1e525ef407295943d7a1c6f62b6 is 50, key is test_row_0/B:col10/1731213562726/Put/seqid=0 2024-11-10T04:39:24,376 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411102990b1978be84474805c1825d06242df_145f65b29b41c5cf34022ad77be79930, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:24,376 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102990b1978be84474805c1825d06242df_145f65b29b41c5cf34022ad77be79930 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742394_1570 (size=12815) 2024-11-10T04:39:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742395_1571 (size=4469) 2024-11-10T04:39:24,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-10T04:39:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,465 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-10T04:39:24,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f120915f86ff476fa7b81d39bf423cf0_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213563580/Put/seqid=0 2024-11-10T04:39:24,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742396_1572 (size=12454) 2024-11-10T04:39:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:24,477 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f120915f86ff476fa7b81d39bf423cf0_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f120915f86ff476fa7b81d39bf423cf0_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:24,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/68523a111f874cb4b4bf60cd1f892d59, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:24,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/68523a111f874cb4b4bf60cd1f892d59 is 175, key is test_row_0/A:col10/1731213563580/Put/seqid=0 2024-11-10T04:39:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T04:39:24,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742397_1573 (size=31255) 2024-11-10T04:39:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:24,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:24,782 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#A#compaction#480 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:24,782 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/40fd5589923f4e3cbf7819767400c5ec is 175, key is test_row_0/A:col10/1731213562726/Put/seqid=0 2024-11-10T04:39:24,784 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/fff0d1e525ef407295943d7a1c6f62b6 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fff0d1e525ef407295943d7a1c6f62b6 2024-11-10T04:39:24,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742398_1574 (size=31769) 2024-11-10T04:39:24,788 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/B of 145f65b29b41c5cf34022ad77be79930 into fff0d1e525ef407295943d7a1c6f62b6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:24,788 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:24,788 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/B, priority=13, startTime=1731213564367; duration=0sec 2024-11-10T04:39:24,788 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:24,788 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:B 2024-11-10T04:39:24,788 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:24,789 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:24,789 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/C is initiating minor compaction (all files) 2024-11-10T04:39:24,790 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/C in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:24,790 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6ee23b700344d9884c6f3dbd5d0bbe3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/9317f1f9bf8843b4b349d28e6d4f561b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/77f60c6f7c42458880a1d516b5128ac9] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.1 K 2024-11-10T04:39:24,790 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c6ee23b700344d9884c6f3dbd5d0bbe3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731213560628 2024-11-10T04:39:24,790 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9317f1f9bf8843b4b349d28e6d4f561b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731213561340 2024-11-10T04:39:24,791 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f60c6f7c42458880a1d516b5128ac9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731213562723 2024-11-10T04:39:24,796 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:24,796 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/6f9c063f484448ac9a9ed1a6bbae2d41 is 50, key is test_row_0/C:col10/1731213562726/Put/seqid=0 2024-11-10T04:39:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742399_1575 (size=12815) 2024-11-10T04:39:24,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213624798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:24,883 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/68523a111f874cb4b4bf60cd1f892d59 2024-11-10T04:39:24,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213624881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:24,884 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:24,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213624881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:24,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213624881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:24,885 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:24,886 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/8cfde47dce524716b9cc782b223a9731 is 50, key is test_row_0/B:col10/1731213563580/Put/seqid=0 2024-11-10T04:39:24,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742400_1576 (size=12301) 2024-11-10T04:39:24,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213624891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:24,895 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_0' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:24,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213624911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T04:39:25,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213625121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:25,190 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/40fd5589923f4e3cbf7819767400c5ec as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/40fd5589923f4e3cbf7819767400c5ec 2024-11-10T04:39:25,194 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/A of 145f65b29b41c5cf34022ad77be79930 into 40fd5589923f4e3cbf7819767400c5ec(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:25,194 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:25,194 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/A, priority=13, startTime=1731213564367; duration=0sec 2024-11-10T04:39:25,194 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:25,194 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:A 2024-11-10T04:39:25,205 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/6f9c063f484448ac9a9ed1a6bbae2d41 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6f9c063f484448ac9a9ed1a6bbae2d41 2024-11-10T04:39:25,209 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/C of 145f65b29b41c5cf34022ad77be79930 into 6f9c063f484448ac9a9ed1a6bbae2d41(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:25,209 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:25,209 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/C, priority=13, startTime=1731213564367; duration=0sec 2024-11-10T04:39:25,209 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:25,209 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:25,294 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/8cfde47dce524716b9cc782b223a9731 2024-11-10T04:39:25,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/fd3f8c0e87b146e9a7eff7e759368c34 is 50, key is test_row_0/C:col10/1731213563580/Put/seqid=0 2024-11-10T04:39:25,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742401_1577 (size=12301) 2024-11-10T04:39:25,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213625431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:25,705 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/fd3f8c0e87b146e9a7eff7e759368c34 2024-11-10T04:39:25,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/68523a111f874cb4b4bf60cd1f892d59 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/68523a111f874cb4b4bf60cd1f892d59 2024-11-10T04:39:25,712 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/68523a111f874cb4b4bf60cd1f892d59, entries=150, sequenceid=276, filesize=30.5 K 2024-11-10T04:39:25,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/8cfde47dce524716b9cc782b223a9731 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8cfde47dce524716b9cc782b223a9731 2024-11-10T04:39:25,715 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8cfde47dce524716b9cc782b223a9731, entries=150, sequenceid=276, filesize=12.0 K 2024-11-10T04:39:25,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/fd3f8c0e87b146e9a7eff7e759368c34 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/fd3f8c0e87b146e9a7eff7e759368c34 2024-11-10T04:39:25,719 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/fd3f8c0e87b146e9a7eff7e759368c34, entries=150, sequenceid=276, filesize=12.0 K 2024-11-10T04:39:25,719 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 145f65b29b41c5cf34022ad77be79930 in 1254ms, sequenceid=276, compaction requested=false 2024-11-10T04:39:25,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:25,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:25,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-10T04:39:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-10T04:39:25,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-10T04:39:25,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-10T04:39:25,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.8720 sec 2024-11-10T04:39:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:25,943 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:25,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-10T04:39:25,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b8ceb6fb11604c54830e4bc4ac8b3be0_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213565942/Put/seqid=0 2024-11-10T04:39:25,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742402_1578 (size=14994) 2024-11-10T04:39:25,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:25,958 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b8ceb6fb11604c54830e4bc4ac8b3be0_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b8ceb6fb11604c54830e4bc4ac8b3be0_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:25,959 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3fd868f7147740eb8fc0ab3c4813c2a1, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:25,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3fd868f7147740eb8fc0ab3c4813c2a1 is 175, key is test_row_0/A:col10/1731213565942/Put/seqid=0 2024-11-10T04:39:25,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742403_1579 (size=39949) 2024-11-10T04:39:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T04:39:26,001 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:26,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:39:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-10T04:39:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T04:39:26,003 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:39:26,004 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:39:26,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:39:26,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213626011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T04:39:26,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213626121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:26,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T04:39:26,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T04:39:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T04:39:26,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213626331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:26,365 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3fd868f7147740eb8fc0ab3c4813c2a1 2024-11-10T04:39:26,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/1a7fb714960446c2bc9e75e2c1c11b0b is 50, key is test_row_0/B:col10/1731213565942/Put/seqid=0 2024-11-10T04:39:26,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742404_1580 (size=12301) 2024-11-10T04:39:26,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T04:39:26,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:26,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T04:39:26,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:26,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T04:39:26,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213626641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:26,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T04:39:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/1a7fb714960446c2bc9e75e2c1c11b0b 2024-11-10T04:39:26,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/1cdd689b4c8d42f08fb93d12e45ec43e is 50, key is test_row_0/C:col10/1731213565942/Put/seqid=0 2024-11-10T04:39:26,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742405_1581 (size=12301) 2024-11-10T04:39:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213626900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213626901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:26,904 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:26,904 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:26,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213626911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:26,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213626911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:26,914 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:26,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:26,914 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_0' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:26,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:26,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:26,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T04:39:26,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:26,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:27,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T04:39:27,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:27,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:27,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:27,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:27,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T04:39:27,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213627151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:27,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/1cdd689b4c8d42f08fb93d12e45ec43e 2024-11-10T04:39:27,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3fd868f7147740eb8fc0ab3c4813c2a1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3fd868f7147740eb8fc0ab3c4813c2a1 2024-11-10T04:39:27,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3fd868f7147740eb8fc0ab3c4813c2a1, entries=200, sequenceid=297, filesize=39.0 K 2024-11-10T04:39:27,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/1a7fb714960446c2bc9e75e2c1c11b0b as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/1a7fb714960446c2bc9e75e2c1c11b0b 2024-11-10T04:39:27,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/1a7fb714960446c2bc9e75e2c1c11b0b, entries=150, sequenceid=297, filesize=12.0 K 2024-11-10T04:39:27,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/1cdd689b4c8d42f08fb93d12e45ec43e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/1cdd689b4c8d42f08fb93d12e45ec43e 2024-11-10T04:39:27,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/1cdd689b4c8d42f08fb93d12e45ec43e, entries=150, sequenceid=297, filesize=12.0 K 2024-11-10T04:39:27,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 145f65b29b41c5cf34022ad77be79930 in 1263ms, sequenceid=297, compaction requested=true 2024-11-10T04:39:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:27,207 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:27,207 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:27,208 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102973 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:27,208 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:27,208 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/A is initiating minor compaction (all files) 2024-11-10T04:39:27,208 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/B is initiating minor compaction (all files) 2024-11-10T04:39:27,208 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/B in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:27,208 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/A in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:27,208 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/40fd5589923f4e3cbf7819767400c5ec, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/68523a111f874cb4b4bf60cd1f892d59, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3fd868f7147740eb8fc0ab3c4813c2a1] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=100.6 K 2024-11-10T04:39:27,208 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fff0d1e525ef407295943d7a1c6f62b6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8cfde47dce524716b9cc782b223a9731, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/1a7fb714960446c2bc9e75e2c1c11b0b] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.5 K 2024-11-10T04:39:27,208 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:27,208 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/40fd5589923f4e3cbf7819767400c5ec, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/68523a111f874cb4b4bf60cd1f892d59, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3fd868f7147740eb8fc0ab3c4813c2a1] 2024-11-10T04:39:27,208 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fff0d1e525ef407295943d7a1c6f62b6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731213562723 2024-11-10T04:39:27,209 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40fd5589923f4e3cbf7819767400c5ec, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731213562723 2024-11-10T04:39:27,209 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cfde47dce524716b9cc782b223a9731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731213563564 2024-11-10T04:39:27,209 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68523a111f874cb4b4bf60cd1f892d59, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731213563564 2024-11-10T04:39:27,209 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a7fb714960446c2bc9e75e2c1c11b0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731213564789 2024-11-10T04:39:27,209 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fd868f7147740eb8fc0ab3c4813c2a1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731213564777 2024-11-10T04:39:27,216 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:27,217 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/23f784e15d5745a3b821c0552ac4aa77 is 50, key is test_row_0/B:col10/1731213565942/Put/seqid=0 2024-11-10T04:39:27,218 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:27,220 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100fb04d212cdf4349b243bbaaac703b64_145f65b29b41c5cf34022ad77be79930 store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:27,221 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100fb04d212cdf4349b243bbaaac703b64_145f65b29b41c5cf34022ad77be79930, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:27,221 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100fb04d212cdf4349b243bbaaac703b64_145f65b29b41c5cf34022ad77be79930 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:27,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742406_1582 (size=13017) 2024-11-10T04:39:27,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742407_1583 (size=4469) 2024-11-10T04:39:27,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-10T04:39:27,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:27,227 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-10T04:39:27,227 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#A#compaction#490 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:27,228 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/948c288d750e41fda191597d38c9a570 is 175, key is test_row_0/A:col10/1731213565942/Put/seqid=0 2024-11-10T04:39:27,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742408_1584 (size=31971) 2024-11-10T04:39:27,234 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/23f784e15d5745a3b821c0552ac4aa77 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/23f784e15d5745a3b821c0552ac4aa77 2024-11-10T04:39:27,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b8c9a73021844bb3b69e762536ae79b6_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213565988/Put/seqid=0 2024-11-10T04:39:27,239 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/B of 145f65b29b41c5cf34022ad77be79930 into 23f784e15d5745a3b821c0552ac4aa77(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:27,239 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:27,239 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/B, priority=13, startTime=1731213567207; duration=0sec 2024-11-10T04:39:27,239 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:27,239 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:B 2024-11-10T04:39:27,239 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:27,240 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:27,240 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/C is initiating minor compaction (all files) 2024-11-10T04:39:27,240 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/C in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:27,240 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6f9c063f484448ac9a9ed1a6bbae2d41, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/fd3f8c0e87b146e9a7eff7e759368c34, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/1cdd689b4c8d42f08fb93d12e45ec43e] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.5 K 2024-11-10T04:39:27,241 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f9c063f484448ac9a9ed1a6bbae2d41, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731213562723 2024-11-10T04:39:27,241 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fd3f8c0e87b146e9a7eff7e759368c34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731213563564 2024-11-10T04:39:27,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742409_1585 (size=12454) 2024-11-10T04:39:27,241 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cdd689b4c8d42f08fb93d12e45ec43e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731213564789 2024-11-10T04:39:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:27,245 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b8c9a73021844bb3b69e762536ae79b6_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b8c9a73021844bb3b69e762536ae79b6_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:27,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/8297331a95124c5f98ed0f8e49066e45, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:27,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/8297331a95124c5f98ed0f8e49066e45 is 175, key is test_row_0/A:col10/1731213565988/Put/seqid=0 2024-11-10T04:39:27,247 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:27,248 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/720215e05dee482b8987b04c75ad9398 is 50, key is test_row_0/C:col10/1731213565942/Put/seqid=0 2024-11-10T04:39:27,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742410_1586 (size=31255) 2024-11-10T04:39:27,251 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/8297331a95124c5f98ed0f8e49066e45 2024-11-10T04:39:27,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742411_1587 (size=13017) 2024-11-10T04:39:27,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/7645713aecc94e7ba98cfacaa0ff2430 is 50, key is test_row_0/B:col10/1731213565988/Put/seqid=0 2024-11-10T04:39:27,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742412_1588 (size=12301) 2024-11-10T04:39:27,641 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/948c288d750e41fda191597d38c9a570 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/948c288d750e41fda191597d38c9a570 2024-11-10T04:39:27,646 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/A of 145f65b29b41c5cf34022ad77be79930 into 948c288d750e41fda191597d38c9a570(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:27,646 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:27,646 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/A, priority=13, startTime=1731213567207; duration=0sec 2024-11-10T04:39:27,646 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:27,646 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:A 2024-11-10T04:39:27,655 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/720215e05dee482b8987b04c75ad9398 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/720215e05dee482b8987b04c75ad9398 2024-11-10T04:39:27,659 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/C of 145f65b29b41c5cf34022ad77be79930 into 720215e05dee482b8987b04c75ad9398(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:27,659 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:27,659 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/C, priority=13, startTime=1731213567207; duration=0sec 2024-11-10T04:39:27,660 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:27,660 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:27,661 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/7645713aecc94e7ba98cfacaa0ff2430 2024-11-10T04:39:27,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/2ea393666e8a43feaee9c33fce99c67f is 50, key is test_row_0/C:col10/1731213565988/Put/seqid=0 2024-11-10T04:39:27,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742413_1589 (size=12301) 2024-11-10T04:39:28,072 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/2ea393666e8a43feaee9c33fce99c67f 2024-11-10T04:39:28,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/8297331a95124c5f98ed0f8e49066e45 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8297331a95124c5f98ed0f8e49066e45 2024-11-10T04:39:28,079 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8297331a95124c5f98ed0f8e49066e45, entries=150, sequenceid=315, filesize=30.5 K 2024-11-10T04:39:28,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/7645713aecc94e7ba98cfacaa0ff2430 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/7645713aecc94e7ba98cfacaa0ff2430 2024-11-10T04:39:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,083 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/7645713aecc94e7ba98cfacaa0ff2430, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T04:39:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/2ea393666e8a43feaee9c33fce99c67f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2ea393666e8a43feaee9c33fce99c67f 2024-11-10T04:39:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,088 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2ea393666e8a43feaee9c33fce99c67f, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T04:39:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,088 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 145f65b29b41c5cf34022ad77be79930 in 861ms, sequenceid=315, compaction requested=false 2024-11-10T04:39:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:28,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-10T04:39:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-10T04:39:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-10T04:39:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0860 sec 2024-11-10T04:39:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.0900 sec 2024-11-10T04:39:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-10T04:39:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,152 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T04:39:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-10T04:39:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T04:39:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,154 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T04:39:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,155 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T04:39:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T04:39:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:28,240 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:28,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T04:39:28,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103f70983655a84566ac1126e5afab2286_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213568233/Put/seqid=0 2024-11-10T04:39:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T04:39:28,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742414_1590 (size=14994) 2024-11-10T04:39:28,262 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:28,266 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103f70983655a84566ac1126e5afab2286_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103f70983655a84566ac1126e5afab2286_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:28,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3f71003f8262450ebe9d7124ea0197e7, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:28,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3f71003f8262450ebe9d7124ea0197e7 is 175, key is test_row_0/A:col10/1731213568233/Put/seqid=0 2024-11-10T04:39:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742415_1591 (size=39949) 2024-11-10T04:39:28,272 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3f71003f8262450ebe9d7124ea0197e7 2024-11-10T04:39:28,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/02f123c21ad04628abef78c5e5f3b67e is 50, key is test_row_0/B:col10/1731213568233/Put/seqid=0 2024-11-10T04:39:28,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742416_1592 (size=12301) 2024-11-10T04:39:28,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T04:39:28,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:28,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213628354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:28,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T04:39:28,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:28,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213628461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:28,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:28,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:28,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T04:39:28,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T04:39:28,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:28,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213628671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:28,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/02f123c21ad04628abef78c5e5f3b67e 2024-11-10T04:39:28,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/268dd0f1625a46bc9223da7dc22ef368 is 50, key is test_row_0/C:col10/1731213568233/Put/seqid=0 2024-11-10T04:39:28,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742417_1593 (size=12301) 2024-11-10T04:39:28,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T04:39:28,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:28,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T04:39:28,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T04:39:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:28,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:28,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213628981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:29,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T04:39:29,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:29,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:29,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:29,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:29,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:39:29,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/268dd0f1625a46bc9223da7dc22ef368 2024-11-10T04:39:29,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/3f71003f8262450ebe9d7124ea0197e7 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3f71003f8262450ebe9d7124ea0197e7 2024-11-10T04:39:29,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3f71003f8262450ebe9d7124ea0197e7, entries=200, sequenceid=329, filesize=39.0 K 2024-11-10T04:39:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/02f123c21ad04628abef78c5e5f3b67e as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/02f123c21ad04628abef78c5e5f3b67e 2024-11-10T04:39:29,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/02f123c21ad04628abef78c5e5f3b67e, entries=150, sequenceid=329, filesize=12.0 K 2024-11-10T04:39:29,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/268dd0f1625a46bc9223da7dc22ef368 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/268dd0f1625a46bc9223da7dc22ef368 2024-11-10T04:39:29,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/268dd0f1625a46bc9223da7dc22ef368, entries=150, sequenceid=329, filesize=12.0 K 2024-11-10T04:39:29,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 145f65b29b41c5cf34022ad77be79930 in 875ms, sequenceid=329, compaction requested=true 2024-11-10T04:39:29,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:29,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:29,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:29,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:29,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:29,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:29,116 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:29,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:29,116 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:29,116 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:29,116 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:29,117 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/A is initiating minor compaction (all files) 2024-11-10T04:39:29,117 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/B is initiating minor compaction (all files) 2024-11-10T04:39:29,117 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/A in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:29,117 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/B in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:29,117 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/948c288d750e41fda191597d38c9a570, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8297331a95124c5f98ed0f8e49066e45, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3f71003f8262450ebe9d7124ea0197e7] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=100.8 K 2024-11-10T04:39:29,117 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/23f784e15d5745a3b821c0552ac4aa77, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/7645713aecc94e7ba98cfacaa0ff2430, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/02f123c21ad04628abef78c5e5f3b67e] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.7 K 2024-11-10T04:39:29,117 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:29,117 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/948c288d750e41fda191597d38c9a570, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8297331a95124c5f98ed0f8e49066e45, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3f71003f8262450ebe9d7124ea0197e7] 2024-11-10T04:39:29,117 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 948c288d750e41fda191597d38c9a570, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731213564789 2024-11-10T04:39:29,117 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f784e15d5745a3b821c0552ac4aa77, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731213564789 2024-11-10T04:39:29,117 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8297331a95124c5f98ed0f8e49066e45, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731213565988 2024-11-10T04:39:29,117 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7645713aecc94e7ba98cfacaa0ff2430, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731213565988 2024-11-10T04:39:29,118 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f71003f8262450ebe9d7124ea0197e7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731213568220 2024-11-10T04:39:29,118 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f123c21ad04628abef78c5e5f3b67e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731213568220 2024-11-10T04:39:29,130 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:29,131 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:29,132 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/fd3b992e313a4447853bce3abab61b7c is 50, key is test_row_0/B:col10/1731213568233/Put/seqid=0 2024-11-10T04:39:29,132 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110900c78f374ff4f0086d6f5ff70833156_145f65b29b41c5cf34022ad77be79930 store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:29,134 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110900c78f374ff4f0086d6f5ff70833156_145f65b29b41c5cf34022ad77be79930, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:29,134 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110900c78f374ff4f0086d6f5ff70833156_145f65b29b41c5cf34022ad77be79930 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:29,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742418_1594 (size=13119) 2024-11-10T04:39:29,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742419_1595 (size=4469) 2024-11-10T04:39:29,141 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#A#compaction#499 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:29,141 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/52db85ae0ff0487fbb6031b0eeaac298 is 175, key is test_row_0/A:col10/1731213568233/Put/seqid=0 2024-11-10T04:39:29,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742420_1596 (size=32073) 2024-11-10T04:39:29,145 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/fd3b992e313a4447853bce3abab61b7c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fd3b992e313a4447853bce3abab61b7c 2024-11-10T04:39:29,148 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/52db85ae0ff0487fbb6031b0eeaac298 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/52db85ae0ff0487fbb6031b0eeaac298 2024-11-10T04:39:29,149 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/B of 145f65b29b41c5cf34022ad77be79930 into fd3b992e313a4447853bce3abab61b7c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:29,149 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:29,149 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/B, priority=13, startTime=1731213569116; duration=0sec 2024-11-10T04:39:29,150 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:29,150 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:B 2024-11-10T04:39:29,150 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:29,151 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:29,151 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/C is initiating minor compaction (all files) 2024-11-10T04:39:29,151 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/C in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:29,151 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/720215e05dee482b8987b04c75ad9398, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2ea393666e8a43feaee9c33fce99c67f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/268dd0f1625a46bc9223da7dc22ef368] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.7 K 2024-11-10T04:39:29,151 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 720215e05dee482b8987b04c75ad9398, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731213564789 2024-11-10T04:39:29,152 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ea393666e8a43feaee9c33fce99c67f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731213565988 2024-11-10T04:39:29,152 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 268dd0f1625a46bc9223da7dc22ef368, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731213568220 2024-11-10T04:39:29,153 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/A of 145f65b29b41c5cf34022ad77be79930 into 52db85ae0ff0487fbb6031b0eeaac298(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:29,153 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:29,153 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/A, priority=13, startTime=1731213569115; duration=0sec 2024-11-10T04:39:29,153 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:29,153 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:A 2024-11-10T04:39:29,158 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#C#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:29,158 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/a16fd83b69414686b828bee34f4369e1 is 50, key is test_row_0/C:col10/1731213568233/Put/seqid=0 2024-11-10T04:39:29,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742421_1597 (size=13119) 2024-11-10T04:39:29,170 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/a16fd83b69414686b828bee34f4369e1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/a16fd83b69414686b828bee34f4369e1 2024-11-10T04:39:29,174 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/C of 145f65b29b41c5cf34022ad77be79930 into a16fd83b69414686b828bee34f4369e1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:29,174 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:29,174 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/C, priority=13, startTime=1731213569116; duration=0sec 2024-11-10T04:39:29,174 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:29,174 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:29,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-10T04:39:29,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:29,228 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:39:29,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e9265be055894b499f7d394db40e1111_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213568324/Put/seqid=0 2024-11-10T04:39:29,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742422_1598 (size=12454) 2024-11-10T04:39:29,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:29,243 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e9265be055894b499f7d394db40e1111_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e9265be055894b499f7d394db40e1111_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/c2b10efd9a64419fba3f7bdea61fb165, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/c2b10efd9a64419fba3f7bdea61fb165 is 175, key is test_row_0/A:col10/1731213568324/Put/seqid=0 2024-11-10T04:39:29,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742423_1599 (size=31255) 2024-11-10T04:39:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T04:39:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:29,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. as already flushing 2024-11-10T04:39:29,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213629520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:29,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:29,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:29,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:29,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213629631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:29,649 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/c2b10efd9a64419fba3f7bdea61fb165 2024-11-10T04:39:29,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/3e476ffad709475e9e705bb33d43b51c is 50, key is test_row_0/B:col10/1731213568324/Put/seqid=0 2024-11-10T04:39:29,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742424_1600 (size=12301) 2024-11-10T04:39:29,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213629841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:30,060 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/3e476ffad709475e9e705bb33d43b51c 2024-11-10T04:39:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/ffaa97701dff4e4fabbb2621782c20d3 is 50, key is test_row_0/C:col10/1731213568324/Put/seqid=0 2024-11-10T04:39:30,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742425_1601 (size=12301) 2024-11-10T04:39:30,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213630151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:30,307 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T04:39:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T04:39:30,478 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/ffaa97701dff4e4fabbb2621782c20d3 2024-11-10T04:39:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/c2b10efd9a64419fba3f7bdea61fb165 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c2b10efd9a64419fba3f7bdea61fb165 2024-11-10T04:39:30,487 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c2b10efd9a64419fba3f7bdea61fb165, entries=150, sequenceid=357, filesize=30.5 K 2024-11-10T04:39:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/3e476ffad709475e9e705bb33d43b51c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/3e476ffad709475e9e705bb33d43b51c 2024-11-10T04:39:30,494 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/3e476ffad709475e9e705bb33d43b51c, entries=150, sequenceid=357, filesize=12.0 K 2024-11-10T04:39:30,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/ffaa97701dff4e4fabbb2621782c20d3 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ffaa97701dff4e4fabbb2621782c20d3 2024-11-10T04:39:30,498 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ffaa97701dff4e4fabbb2621782c20d3, entries=150, sequenceid=357, filesize=12.0 K 2024-11-10T04:39:30,499 INFO [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 145f65b29b41c5cf34022ad77be79930 in 1271ms, sequenceid=357, compaction requested=false 2024-11-10T04:39:30,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:30,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:30,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d1d9e3766cb1:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-10T04:39:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-10T04:39:30,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-10T04:39:30,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3470 sec 2024-11-10T04:39:30,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.3550 sec 2024-11-10T04:39:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:30,663 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:30,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T04:39:30,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111026cd40af7cc84d038b175ef39c9f0076_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213569503/Put/seqid=0 2024-11-10T04:39:30,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742426_1602 (size=14994) 2024-11-10T04:39:30,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213630765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:30,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213630881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:30,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39384 deadline: 1731213630911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:30,915 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:30,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39352 deadline: 1731213630951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:30,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39368 deadline: 1731213630952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:30,954 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,955 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_2' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:30,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39364 deadline: 1731213630961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:30,964 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to d1d9e3766cb1,45693,1731213391842 for 'test_row_0' in 145f65b29b41c5cf34022ad77be79930 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-10T04:39:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:31,091 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:31,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213631091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:31,094 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111026cd40af7cc84d038b175ef39c9f0076_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111026cd40af7cc84d038b175ef39c9f0076_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:31,095 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/b9f86afc01b04dfcabd812e158a7bb40, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:31,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/b9f86afc01b04dfcabd812e158a7bb40 is 175, key is test_row_0/A:col10/1731213569503/Put/seqid=0 2024-11-10T04:39:31,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742427_1603 (size=39949) 2024-11-10T04:39:31,231 INFO [Thread-2253 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2253. 2024-11-10T04:39:31,231 DEBUG [Thread-2253 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:31,231 DEBUG [Thread-2253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,231 DEBUG [Thread-2253 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:31,232 INFO [Thread-2251 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2251. 2024-11-10T04:39:31,232 DEBUG [Thread-2251 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:31,232 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,232 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:31,234 INFO [Thread-2249 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2249. 2024-11-10T04:39:31,234 DEBUG [Thread-2249 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:31,234 DEBUG [Thread-2249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,234 DEBUG [Thread-2249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:31,235 INFO [Thread-2255 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2255. 2024-11-10T04:39:31,235 DEBUG [Thread-2255 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:31,235 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,235 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:31,236 INFO [Thread-2247 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2247. 2024-11-10T04:39:31,236 DEBUG [Thread-2247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:31,236 DEBUG [Thread-2247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,236 DEBUG [Thread-2247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:31,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:31,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213631401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:31,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:31,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:31,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:31,501 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/b9f86afc01b04dfcabd812e158a7bb40 2024-11-10T04:39:31,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/479f9821fa9240debf4178fe44daef67 is 50, key is test_row_0/B:col10/1731213569503/Put/seqid=0 2024-11-10T04:39:31,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742428_1604 (size=12301) 2024-11-10T04:39:31,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/479f9821fa9240debf4178fe44daef67 2024-11-10T04:39:31,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T04:39:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39336 deadline: 1731213631911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 , the old value is region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T04:39:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., hostname=d1d9e3766cb1,45693,1731213391842, seqNum=5 because the exception is null or not the one we care about 2024-11-10T04:39:31,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/4eac081b344e4ef7bbe9deafd58d4f13 is 50, key is test_row_0/C:col10/1731213569503/Put/seqid=0 2024-11-10T04:39:31,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742429_1605 (size=12301) 2024-11-10T04:39:32,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/4eac081b344e4ef7bbe9deafd58d4f13 2024-11-10T04:39:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T04:39:32,322 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:32,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/b9f86afc01b04dfcabd812e158a7bb40 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b9f86afc01b04dfcabd812e158a7bb40 2024-11-10T04:39:32,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b9f86afc01b04dfcabd812e158a7bb40, entries=200, sequenceid=369, filesize=39.0 K 2024-11-10T04:39:32,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/479f9821fa9240debf4178fe44daef67 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/479f9821fa9240debf4178fe44daef67 2024-11-10T04:39:32,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/479f9821fa9240debf4178fe44daef67, entries=150, sequenceid=369, filesize=12.0 K 2024-11-10T04:39:32,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/4eac081b344e4ef7bbe9deafd58d4f13 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/4eac081b344e4ef7bbe9deafd58d4f13 2024-11-10T04:39:32,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/4eac081b344e4ef7bbe9deafd58d4f13, entries=150, sequenceid=369, filesize=12.0 K 2024-11-10T04:39:32,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 145f65b29b41c5cf34022ad77be79930 in 1668ms, sequenceid=369, compaction requested=true 2024-11-10T04:39:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T04:39:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T04:39:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 145f65b29b41c5cf34022ad77be79930:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T04:39:32,331 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:32,331 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:32,332 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:32,332 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:32,332 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/A is initiating minor compaction (all files) 2024-11-10T04:39:32,332 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/B is initiating minor compaction (all files) 2024-11-10T04:39:32,332 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/A in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:32,332 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/B in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:32,332 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/52db85ae0ff0487fbb6031b0eeaac298, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c2b10efd9a64419fba3f7bdea61fb165, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b9f86afc01b04dfcabd812e158a7bb40] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=100.9 K 2024-11-10T04:39:32,332 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fd3b992e313a4447853bce3abab61b7c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/3e476ffad709475e9e705bb33d43b51c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/479f9821fa9240debf4178fe44daef67] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.8 K 2024-11-10T04:39:32,332 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:32,332 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. files: [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/52db85ae0ff0487fbb6031b0eeaac298, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c2b10efd9a64419fba3f7bdea61fb165, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b9f86afc01b04dfcabd812e158a7bb40] 2024-11-10T04:39:32,332 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fd3b992e313a4447853bce3abab61b7c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731213568220 2024-11-10T04:39:32,332 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52db85ae0ff0487fbb6031b0eeaac298, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731213568220 2024-11-10T04:39:32,332 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e476ffad709475e9e705bb33d43b51c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731213568324 2024-11-10T04:39:32,333 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b10efd9a64419fba3f7bdea61fb165, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731213568324 2024-11-10T04:39:32,333 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9f86afc01b04dfcabd812e158a7bb40, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731213569503 2024-11-10T04:39:32,333 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 479f9821fa9240debf4178fe44daef67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731213569503 2024-11-10T04:39:32,337 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:32,338 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#B#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:32,338 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/7695a37b31a14d21a947e259db20e2c2 is 50, key is test_row_0/B:col10/1731213569503/Put/seqid=0 2024-11-10T04:39:32,339 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411102436e7c101764be59f3944a714e7b659_145f65b29b41c5cf34022ad77be79930 store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:32,341 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411102436e7c101764be59f3944a714e7b659_145f65b29b41c5cf34022ad77be79930, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:32,341 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102436e7c101764be59f3944a714e7b659_145f65b29b41c5cf34022ad77be79930 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:32,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742430_1606 (size=13221) 2024-11-10T04:39:32,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742431_1607 (size=4469) 2024-11-10T04:39:32,745 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#A#compaction#508 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:32,746 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/8606f69849e94df49f48d065240399c8 is 175, key is test_row_0/A:col10/1731213569503/Put/seqid=0 2024-11-10T04:39:32,747 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/7695a37b31a14d21a947e259db20e2c2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/7695a37b31a14d21a947e259db20e2c2 2024-11-10T04:39:32,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742432_1608 (size=32175) 2024-11-10T04:39:32,751 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/B of 145f65b29b41c5cf34022ad77be79930 into 7695a37b31a14d21a947e259db20e2c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:32,751 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:32,751 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/B, priority=13, startTime=1731213572331; duration=0sec 2024-11-10T04:39:32,751 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T04:39:32,751 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:B 2024-11-10T04:39:32,751 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T04:39:32,752 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T04:39:32,752 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1541): 145f65b29b41c5cf34022ad77be79930/C is initiating minor compaction (all files) 2024-11-10T04:39:32,752 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 145f65b29b41c5cf34022ad77be79930/C in TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:32,752 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/a16fd83b69414686b828bee34f4369e1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ffaa97701dff4e4fabbb2621782c20d3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/4eac081b344e4ef7bbe9deafd58d4f13] into tmpdir=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp, totalSize=36.8 K 2024-11-10T04:39:32,752 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/8606f69849e94df49f48d065240399c8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8606f69849e94df49f48d065240399c8 2024-11-10T04:39:32,752 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a16fd83b69414686b828bee34f4369e1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731213568220 2024-11-10T04:39:32,753 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ffaa97701dff4e4fabbb2621782c20d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731213568324 2024-11-10T04:39:32,753 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eac081b344e4ef7bbe9deafd58d4f13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731213569503 2024-11-10T04:39:32,756 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/A of 145f65b29b41c5cf34022ad77be79930 into 8606f69849e94df49f48d065240399c8(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:32,756 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:32,756 INFO [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/A, priority=13, startTime=1731213572331; duration=0sec 2024-11-10T04:39:32,756 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:32,756 DEBUG [RS:0;d1d9e3766cb1:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:A 2024-11-10T04:39:32,758 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 145f65b29b41c5cf34022ad77be79930#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T04:39:32,758 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/b550d58fb37843c0ac8ff3476e7cff84 is 50, key is test_row_0/C:col10/1731213569503/Put/seqid=0 2024-11-10T04:39:32,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742433_1609 (size=13221) 2024-11-10T04:39:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:32,923 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T04:39:32,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T04:39:32,923 INFO [Thread-2244 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2244. 2024-11-10T04:39:32,923 DEBUG [Thread-2244 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:32,923 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:32,923 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:32,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:32,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111017e304325d024ae8a6ff6508cc0acb1a_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213572922/Put/seqid=0 2024-11-10T04:39:32,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742434_1610 (size=12454) 2024-11-10T04:39:33,164 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/b550d58fb37843c0ac8ff3476e7cff84 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/b550d58fb37843c0ac8ff3476e7cff84 2024-11-10T04:39:33,168 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 145f65b29b41c5cf34022ad77be79930/C of 145f65b29b41c5cf34022ad77be79930 into b550d58fb37843c0ac8ff3476e7cff84(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T04:39:33,168 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:33,168 INFO [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., storeName=145f65b29b41c5cf34022ad77be79930/C, priority=13, startTime=1731213572331; duration=0sec 2024-11-10T04:39:33,168 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T04:39:33,168 DEBUG [RS:0;d1d9e3766cb1:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 145f65b29b41c5cf34022ad77be79930:C 2024-11-10T04:39:33,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:33,333 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111017e304325d024ae8a6ff6508cc0acb1a_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111017e304325d024ae8a6ff6508cc0acb1a_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:33,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/1edf9f367fa142a4bc8b1fb78c4fa830, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:33,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/1edf9f367fa142a4bc8b1fb78c4fa830 is 175, key is test_row_0/A:col10/1731213572922/Put/seqid=0 2024-11-10T04:39:33,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742435_1611 (size=31255) 2024-11-10T04:39:33,738 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/1edf9f367fa142a4bc8b1fb78c4fa830 2024-11-10T04:39:33,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/8c51fe083d684dabb92d6e94b06d3a82 is 50, key is test_row_0/B:col10/1731213572922/Put/seqid=0 2024-11-10T04:39:33,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742436_1612 (size=12301) 2024-11-10T04:39:34,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/8c51fe083d684dabb92d6e94b06d3a82 2024-11-10T04:39:34,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/88eb8862666f4767ba2f1fae1e12b346 is 50, key is test_row_0/C:col10/1731213572922/Put/seqid=0 2024-11-10T04:39:34,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742437_1613 (size=12301) 2024-11-10T04:39:34,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/88eb8862666f4767ba2f1fae1e12b346 2024-11-10T04:39:34,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/1edf9f367fa142a4bc8b1fb78c4fa830 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/1edf9f367fa142a4bc8b1fb78c4fa830 2024-11-10T04:39:34,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/1edf9f367fa142a4bc8b1fb78c4fa830, entries=150, sequenceid=396, filesize=30.5 K 2024-11-10T04:39:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/8c51fe083d684dabb92d6e94b06d3a82 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8c51fe083d684dabb92d6e94b06d3a82 2024-11-10T04:39:34,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8c51fe083d684dabb92d6e94b06d3a82, entries=150, sequenceid=396, filesize=12.0 K 2024-11-10T04:39:34,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/88eb8862666f4767ba2f1fae1e12b346 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/88eb8862666f4767ba2f1fae1e12b346 2024-11-10T04:39:34,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/88eb8862666f4767ba2f1fae1e12b346, entries=150, sequenceid=396, filesize=12.0 K 2024-11-10T04:39:34,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 145f65b29b41c5cf34022ad77be79930 in 1645ms, sequenceid=396, compaction requested=false 2024-11-10T04:39:34,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 145f65b29b41c5cf34022ad77be79930: 2024-11-10T04:39:40,943 INFO [Thread-2240 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2240. 2024-11-10T04:39:40,943 DEBUG [Thread-2240 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:40,943 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:40,943 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:40,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:40,982 INFO [Thread-2236 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2236. 2024-11-10T04:39:40,982 DEBUG [Thread-2236 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:40,982 DEBUG [Thread-2236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:40,982 DEBUG [Thread-2236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:40,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:41,032 INFO [Thread-2242 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2242. 2024-11-10T04:39:41,032 DEBUG [Thread-2242 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:41,032 DEBUG [Thread-2242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:41,033 DEBUG [Thread-2242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:41,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:41,042 INFO [Thread-2238 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2238. 2024-11-10T04:39:41,042 DEBUG [Thread-2238 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T04:39:41,042 DEBUG [Thread-2238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:41,042 DEBUG [Thread-2238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:41,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 194 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3745 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3373 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3412 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3736 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3388 2024-11-10T04:39:41,042 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T04:39:41,042 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:39:41,042 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:39:41,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:41,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:41,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:41,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T04:39:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T04:39:41,045 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213581045"}]},"ts":"1731213581045"} 2024-11-10T04:39:41,046 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T04:39:41,046 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T04:39:41,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T04:39:41,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, UNASSIGN}] 2024-11-10T04:39:41,049 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, UNASSIGN 2024-11-10T04:39:41,049 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=145f65b29b41c5cf34022ad77be79930, regionState=CLOSING, regionLocation=d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:41,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, UNASSIGN because future has completed 2024-11-10T04:39:41,050 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T04:39:41,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842}] 2024-11-10T04:39:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T04:39:41,202 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:41,202 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T04:39:41,202 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 145f65b29b41c5cf34022ad77be79930, disabling compactions & flushes 2024-11-10T04:39:41,202 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:41,202 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:41,202 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. after waiting 0 ms 2024-11-10T04:39:41,202 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:41,202 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing 145f65b29b41c5cf34022ad77be79930 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-10T04:39:41,207 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dee604829128490c870d36a83cebb505_145f65b29b41c5cf34022ad77be79930 is 50, key is test_row_0/A:col10/1731213581032/Put/seqid=0 2024-11-10T04:39:41,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742438_1614 (size=9914) 2024-11-10T04:39:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T04:39:41,611 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:39:41,614 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dee604829128490c870d36a83cebb505_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dee604829128490c870d36a83cebb505_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:41,614 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/ae5856d3d1a548dab47662bc456fc514, store: [table=TestAcidGuarantees family=A region=145f65b29b41c5cf34022ad77be79930] 2024-11-10T04:39:41,615 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/ae5856d3d1a548dab47662bc456fc514 is 175, key is test_row_0/A:col10/1731213581032/Put/seqid=0 2024-11-10T04:39:41,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742439_1615 (size=22561) 2024-11-10T04:39:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T04:39:42,018 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/ae5856d3d1a548dab47662bc456fc514 2024-11-10T04:39:42,023 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/0f662d52b9ca4678bad2b0f956d1473c is 50, key is test_row_0/B:col10/1731213581032/Put/seqid=0 2024-11-10T04:39:42,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742440_1616 (size=9857) 2024-11-10T04:39:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T04:39:42,427 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/0f662d52b9ca4678bad2b0f956d1473c 2024-11-10T04:39:42,432 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/43bc43f4dc4f4c0bab24a38880bbfd0f is 50, key is test_row_0/C:col10/1731213581032/Put/seqid=0 2024-11-10T04:39:42,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742441_1617 (size=9857) 2024-11-10T04:39:42,835 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/43bc43f4dc4f4c0bab24a38880bbfd0f 2024-11-10T04:39:42,838 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/A/ae5856d3d1a548dab47662bc456fc514 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ae5856d3d1a548dab47662bc456fc514 2024-11-10T04:39:42,841 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ae5856d3d1a548dab47662bc456fc514, entries=100, sequenceid=404, filesize=22.0 K 2024-11-10T04:39:42,841 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/B/0f662d52b9ca4678bad2b0f956d1473c as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/0f662d52b9ca4678bad2b0f956d1473c 2024-11-10T04:39:42,844 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/0f662d52b9ca4678bad2b0f956d1473c, entries=100, sequenceid=404, filesize=9.6 K 2024-11-10T04:39:42,844 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/.tmp/C/43bc43f4dc4f4c0bab24a38880bbfd0f as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/43bc43f4dc4f4c0bab24a38880bbfd0f 2024-11-10T04:39:42,846 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/43bc43f4dc4f4c0bab24a38880bbfd0f, entries=100, sequenceid=404, filesize=9.6 K 2024-11-10T04:39:42,847 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 145f65b29b41c5cf34022ad77be79930 in 1645ms, sequenceid=404, compaction requested=true 2024-11-10T04:39:42,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d146b070f2414e89861454f65ce38c9d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3d3da0d7ff1e4777996ab4cd32b3f4d5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/5c592a7ef59f4ff09c41faab00725294, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/02888f868cee407b87a80eb62c100c6f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/abe15a7e00c14ffd8fa28a1abec40340, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/2b9b51f3cf0c4c1f80cbf3cb5dafc74e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/029ecf7b91c14fdaab7a93406638b25b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/de54ec33b470420fadb5d5df54888506, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/da57cb59c7fa47c597997600192a956e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/45bbaa5ea95749f6a8b6e44adb7797ab, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/9275f8ebf874412cbbae287af6034923, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8bfad6d2b5d54b069d5a7799d039f419, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/01994e9576c44ee9b612478befe31063, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b68ce68cfa5c4d77b348451acd3e3e57, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/bf6c7009200544349ad18f826e9b5cca, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ce6826bd76be4580bb3389135faab171, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c9cebce0e84a4b88ad5d188135883e68, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d200df297fb24f1c8a8c5b4ec1355bc0, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/40fd5589923f4e3cbf7819767400c5ec, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/68523a111f874cb4b4bf60cd1f892d59, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3fd868f7147740eb8fc0ab3c4813c2a1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/948c288d750e41fda191597d38c9a570, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8297331a95124c5f98ed0f8e49066e45, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3f71003f8262450ebe9d7124ea0197e7, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/52db85ae0ff0487fbb6031b0eeaac298, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c2b10efd9a64419fba3f7bdea61fb165, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b9f86afc01b04dfcabd812e158a7bb40] to archive 2024-11-10T04:39:42,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:39:42,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d146b070f2414e89861454f65ce38c9d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d146b070f2414e89861454f65ce38c9d 2024-11-10T04:39:42,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3d3da0d7ff1e4777996ab4cd32b3f4d5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3d3da0d7ff1e4777996ab4cd32b3f4d5 2024-11-10T04:39:42,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/5c592a7ef59f4ff09c41faab00725294 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/5c592a7ef59f4ff09c41faab00725294 2024-11-10T04:39:42,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/02888f868cee407b87a80eb62c100c6f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/02888f868cee407b87a80eb62c100c6f 2024-11-10T04:39:42,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/abe15a7e00c14ffd8fa28a1abec40340 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/abe15a7e00c14ffd8fa28a1abec40340 2024-11-10T04:39:42,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/2b9b51f3cf0c4c1f80cbf3cb5dafc74e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/2b9b51f3cf0c4c1f80cbf3cb5dafc74e 2024-11-10T04:39:42,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/029ecf7b91c14fdaab7a93406638b25b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/029ecf7b91c14fdaab7a93406638b25b 2024-11-10T04:39:42,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/de54ec33b470420fadb5d5df54888506 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/de54ec33b470420fadb5d5df54888506 2024-11-10T04:39:42,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/da57cb59c7fa47c597997600192a956e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/da57cb59c7fa47c597997600192a956e 2024-11-10T04:39:42,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/45bbaa5ea95749f6a8b6e44adb7797ab to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/45bbaa5ea95749f6a8b6e44adb7797ab 2024-11-10T04:39:42,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/9275f8ebf874412cbbae287af6034923 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/9275f8ebf874412cbbae287af6034923 2024-11-10T04:39:42,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8bfad6d2b5d54b069d5a7799d039f419 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8bfad6d2b5d54b069d5a7799d039f419 2024-11-10T04:39:42,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/01994e9576c44ee9b612478befe31063 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/01994e9576c44ee9b612478befe31063 2024-11-10T04:39:42,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b68ce68cfa5c4d77b348451acd3e3e57 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b68ce68cfa5c4d77b348451acd3e3e57 2024-11-10T04:39:42,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/bf6c7009200544349ad18f826e9b5cca to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/bf6c7009200544349ad18f826e9b5cca 2024-11-10T04:39:42,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ce6826bd76be4580bb3389135faab171 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ce6826bd76be4580bb3389135faab171 2024-11-10T04:39:42,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c9cebce0e84a4b88ad5d188135883e68 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c9cebce0e84a4b88ad5d188135883e68 2024-11-10T04:39:42,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d200df297fb24f1c8a8c5b4ec1355bc0 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/d200df297fb24f1c8a8c5b4ec1355bc0 2024-11-10T04:39:42,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/40fd5589923f4e3cbf7819767400c5ec to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/40fd5589923f4e3cbf7819767400c5ec 2024-11-10T04:39:42,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/68523a111f874cb4b4bf60cd1f892d59 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/68523a111f874cb4b4bf60cd1f892d59 2024-11-10T04:39:42,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3fd868f7147740eb8fc0ab3c4813c2a1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3fd868f7147740eb8fc0ab3c4813c2a1 2024-11-10T04:39:42,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/948c288d750e41fda191597d38c9a570 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/948c288d750e41fda191597d38c9a570 2024-11-10T04:39:42,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8297331a95124c5f98ed0f8e49066e45 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8297331a95124c5f98ed0f8e49066e45 2024-11-10T04:39:42,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3f71003f8262450ebe9d7124ea0197e7 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/3f71003f8262450ebe9d7124ea0197e7 2024-11-10T04:39:42,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/52db85ae0ff0487fbb6031b0eeaac298 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/52db85ae0ff0487fbb6031b0eeaac298 2024-11-10T04:39:42,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c2b10efd9a64419fba3f7bdea61fb165 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/c2b10efd9a64419fba3f7bdea61fb165 2024-11-10T04:39:42,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b9f86afc01b04dfcabd812e158a7bb40 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/b9f86afc01b04dfcabd812e158a7bb40 2024-11-10T04:39:42,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e4be4a1810ac4e28a00e5cac666a1041, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/b9613766772c40ff85b01aae57f97a7f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/2ec7689a7f904f288705ea3db7f93d14, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e97c973643d4425fa27286b20f984101, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/c09cb8bee781447b947795cda235a8ce, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/85770e356ff44a0985518e6cdc224565, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/38377ad501ca41dd9b2fe47fd30f7673, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/33842d9d53ab470f87bac1a299b70d74, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/991d8a6babd8464d86ce161d7a6a71d4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/def86dbfe1e94d05a431b335d1e492f9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4283d8cd5b6a497a9544eb854d00629c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/84c7fde6e6554eadb0e671a6d298dc92, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/797c6fd27da04409b59ea314229acc8d, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/ff6bd64ca21a4d838cb7141ff36290e5, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/6dee58aea49b4bbf96e47d77623c1709, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/64ab41c6ccf04edd9c114fc6bb9cf7ff, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/05ca1f13459e4345bdd3b7f214201588, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fff0d1e525ef407295943d7a1c6f62b6, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4e521da0cb65495fb791a532cff9e244, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8cfde47dce524716b9cc782b223a9731, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/23f784e15d5745a3b821c0552ac4aa77, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/1a7fb714960446c2bc9e75e2c1c11b0b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/7645713aecc94e7ba98cfacaa0ff2430, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fd3b992e313a4447853bce3abab61b7c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/02f123c21ad04628abef78c5e5f3b67e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/3e476ffad709475e9e705bb33d43b51c, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/479f9821fa9240debf4178fe44daef67] to archive 2024-11-10T04:39:42,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:39:42,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e4be4a1810ac4e28a00e5cac666a1041 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e4be4a1810ac4e28a00e5cac666a1041 2024-11-10T04:39:42,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/b9613766772c40ff85b01aae57f97a7f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/b9613766772c40ff85b01aae57f97a7f 2024-11-10T04:39:42,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/2ec7689a7f904f288705ea3db7f93d14 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/2ec7689a7f904f288705ea3db7f93d14 2024-11-10T04:39:42,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e97c973643d4425fa27286b20f984101 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/e97c973643d4425fa27286b20f984101 2024-11-10T04:39:42,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/c09cb8bee781447b947795cda235a8ce to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/c09cb8bee781447b947795cda235a8ce 2024-11-10T04:39:42,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/85770e356ff44a0985518e6cdc224565 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/85770e356ff44a0985518e6cdc224565 2024-11-10T04:39:42,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/38377ad501ca41dd9b2fe47fd30f7673 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/38377ad501ca41dd9b2fe47fd30f7673 2024-11-10T04:39:42,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/33842d9d53ab470f87bac1a299b70d74 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/33842d9d53ab470f87bac1a299b70d74 2024-11-10T04:39:42,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/991d8a6babd8464d86ce161d7a6a71d4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/991d8a6babd8464d86ce161d7a6a71d4 2024-11-10T04:39:42,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/def86dbfe1e94d05a431b335d1e492f9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/def86dbfe1e94d05a431b335d1e492f9 2024-11-10T04:39:42,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4283d8cd5b6a497a9544eb854d00629c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4283d8cd5b6a497a9544eb854d00629c 2024-11-10T04:39:42,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/84c7fde6e6554eadb0e671a6d298dc92 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/84c7fde6e6554eadb0e671a6d298dc92 2024-11-10T04:39:42,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/797c6fd27da04409b59ea314229acc8d to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/797c6fd27da04409b59ea314229acc8d 2024-11-10T04:39:42,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/ff6bd64ca21a4d838cb7141ff36290e5 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/ff6bd64ca21a4d838cb7141ff36290e5 2024-11-10T04:39:42,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/6dee58aea49b4bbf96e47d77623c1709 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/6dee58aea49b4bbf96e47d77623c1709 2024-11-10T04:39:42,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/64ab41c6ccf04edd9c114fc6bb9cf7ff to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/64ab41c6ccf04edd9c114fc6bb9cf7ff 2024-11-10T04:39:42,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/05ca1f13459e4345bdd3b7f214201588 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/05ca1f13459e4345bdd3b7f214201588 2024-11-10T04:39:42,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fff0d1e525ef407295943d7a1c6f62b6 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fff0d1e525ef407295943d7a1c6f62b6 2024-11-10T04:39:42,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4e521da0cb65495fb791a532cff9e244 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/4e521da0cb65495fb791a532cff9e244 2024-11-10T04:39:42,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8cfde47dce524716b9cc782b223a9731 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8cfde47dce524716b9cc782b223a9731 2024-11-10T04:39:42,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/23f784e15d5745a3b821c0552ac4aa77 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/23f784e15d5745a3b821c0552ac4aa77 2024-11-10T04:39:42,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/1a7fb714960446c2bc9e75e2c1c11b0b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/1a7fb714960446c2bc9e75e2c1c11b0b 2024-11-10T04:39:42,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/7645713aecc94e7ba98cfacaa0ff2430 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/7645713aecc94e7ba98cfacaa0ff2430 2024-11-10T04:39:42,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fd3b992e313a4447853bce3abab61b7c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/fd3b992e313a4447853bce3abab61b7c 2024-11-10T04:39:42,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/02f123c21ad04628abef78c5e5f3b67e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/02f123c21ad04628abef78c5e5f3b67e 2024-11-10T04:39:42,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/3e476ffad709475e9e705bb33d43b51c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/3e476ffad709475e9e705bb33d43b51c 2024-11-10T04:39:42,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/479f9821fa9240debf4178fe44daef67 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/479f9821fa9240debf4178fe44daef67 2024-11-10T04:39:42,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/e8bca83d804e41eb99668c2263b73880, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2a4d5ff345ec4be9a42243d158c1cc43, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/eef2c64485ad4563bc7e3f7debb111cd, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ed0f09b5df654078a11453bb6364dfc4, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/0303f26318f4461ca129d88d6169e09f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/59a52f8315eb448b981ca7a832dbe287, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6eae7b6697d3440689032bfbfac410ae, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/7894a5e074694c1881f4f9a5d7f7a437, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6d350d9a73047f69881e8951cd9b686, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/262512bfe883410ea7fae432e7c3f851, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0f2c3682f424c97a53d609ca7fda553, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0e939a887a84570bc37a2e2aa439b07, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/dd22068f46da46b6b874d656bec341eb, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6a57fc76e80b41c08047dcc64f479bc1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6ee23b700344d9884c6f3dbd5d0bbe3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/79923df1ac814d728a78a4cc0a4583de, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/9317f1f9bf8843b4b349d28e6d4f561b, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6f9c063f484448ac9a9ed1a6bbae2d41, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/77f60c6f7c42458880a1d516b5128ac9, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/fd3f8c0e87b146e9a7eff7e759368c34, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/720215e05dee482b8987b04c75ad9398, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/1cdd689b4c8d42f08fb93d12e45ec43e, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2ea393666e8a43feaee9c33fce99c67f, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/a16fd83b69414686b828bee34f4369e1, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/268dd0f1625a46bc9223da7dc22ef368, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ffaa97701dff4e4fabbb2621782c20d3, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/4eac081b344e4ef7bbe9deafd58d4f13] to archive 2024-11-10T04:39:42,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T04:39:42,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/e8bca83d804e41eb99668c2263b73880 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/e8bca83d804e41eb99668c2263b73880 2024-11-10T04:39:42,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2a4d5ff345ec4be9a42243d158c1cc43 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2a4d5ff345ec4be9a42243d158c1cc43 2024-11-10T04:39:42,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/eef2c64485ad4563bc7e3f7debb111cd to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/eef2c64485ad4563bc7e3f7debb111cd 2024-11-10T04:39:42,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ed0f09b5df654078a11453bb6364dfc4 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ed0f09b5df654078a11453bb6364dfc4 2024-11-10T04:39:42,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/0303f26318f4461ca129d88d6169e09f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/0303f26318f4461ca129d88d6169e09f 2024-11-10T04:39:42,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/59a52f8315eb448b981ca7a832dbe287 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/59a52f8315eb448b981ca7a832dbe287 2024-11-10T04:39:42,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6eae7b6697d3440689032bfbfac410ae to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6eae7b6697d3440689032bfbfac410ae 2024-11-10T04:39:42,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/7894a5e074694c1881f4f9a5d7f7a437 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/7894a5e074694c1881f4f9a5d7f7a437 2024-11-10T04:39:42,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6d350d9a73047f69881e8951cd9b686 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6d350d9a73047f69881e8951cd9b686 2024-11-10T04:39:42,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/262512bfe883410ea7fae432e7c3f851 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/262512bfe883410ea7fae432e7c3f851 2024-11-10T04:39:42,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0f2c3682f424c97a53d609ca7fda553 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0f2c3682f424c97a53d609ca7fda553 2024-11-10T04:39:42,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0e939a887a84570bc37a2e2aa439b07 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/d0e939a887a84570bc37a2e2aa439b07 2024-11-10T04:39:42,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/dd22068f46da46b6b874d656bec341eb to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/dd22068f46da46b6b874d656bec341eb 2024-11-10T04:39:42,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6a57fc76e80b41c08047dcc64f479bc1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6a57fc76e80b41c08047dcc64f479bc1 2024-11-10T04:39:42,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6ee23b700344d9884c6f3dbd5d0bbe3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/c6ee23b700344d9884c6f3dbd5d0bbe3 2024-11-10T04:39:42,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/79923df1ac814d728a78a4cc0a4583de to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/79923df1ac814d728a78a4cc0a4583de 2024-11-10T04:39:42,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/9317f1f9bf8843b4b349d28e6d4f561b to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/9317f1f9bf8843b4b349d28e6d4f561b 2024-11-10T04:39:42,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6f9c063f484448ac9a9ed1a6bbae2d41 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/6f9c063f484448ac9a9ed1a6bbae2d41 2024-11-10T04:39:42,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/77f60c6f7c42458880a1d516b5128ac9 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/77f60c6f7c42458880a1d516b5128ac9 2024-11-10T04:39:42,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/fd3f8c0e87b146e9a7eff7e759368c34 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/fd3f8c0e87b146e9a7eff7e759368c34 2024-11-10T04:39:42,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/720215e05dee482b8987b04c75ad9398 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/720215e05dee482b8987b04c75ad9398 2024-11-10T04:39:42,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/1cdd689b4c8d42f08fb93d12e45ec43e to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/1cdd689b4c8d42f08fb93d12e45ec43e 2024-11-10T04:39:42,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2ea393666e8a43feaee9c33fce99c67f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/2ea393666e8a43feaee9c33fce99c67f 2024-11-10T04:39:42,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/a16fd83b69414686b828bee34f4369e1 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/a16fd83b69414686b828bee34f4369e1 2024-11-10T04:39:42,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/268dd0f1625a46bc9223da7dc22ef368 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/268dd0f1625a46bc9223da7dc22ef368 2024-11-10T04:39:42,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ffaa97701dff4e4fabbb2621782c20d3 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/ffaa97701dff4e4fabbb2621782c20d3 2024-11-10T04:39:42,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/4eac081b344e4ef7bbe9deafd58d4f13 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/4eac081b344e4ef7bbe9deafd58d4f13 2024-11-10T04:39:42,916 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/recovered.edits/407.seqid, newMaxSeqId=407, maxSeqId=4 2024-11-10T04:39:42,916 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. 2024-11-10T04:39:42,917 DEBUG [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 145f65b29b41c5cf34022ad77be79930: Waiting for close lock at 1731213581202Running coprocessor pre-close hooks at 1731213581202Disabling compacts and flushes for region at 1731213581202Disabling writes for close at 1731213581202Obtaining lock to block concurrent updates at 1731213581202Preparing flush snapshotting stores in 145f65b29b41c5cf34022ad77be79930 at 1731213581202Finished memstore snapshotting TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731213581203 (+1 ms)Flushing stores of TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930. at 1731213581203Flushing 145f65b29b41c5cf34022ad77be79930/A: creating writer at 1731213581203Flushing mob file 145f65b29b41c5cf34022ad77be79930/A: appending metadata at 1731213581207 (+4 ms)Flushing mob file 145f65b29b41c5cf34022ad77be79930/A: closing flushed file at 1731213581207Flushing 145f65b29b41c5cf34022ad77be79930/A: appending metadata at 1731213581615 (+408 ms)Flushing 145f65b29b41c5cf34022ad77be79930/A: closing flushed file at 1731213581615Flushing 145f65b29b41c5cf34022ad77be79930/B: creating writer at 1731213582021 (+406 ms)Flushing 145f65b29b41c5cf34022ad77be79930/B: appending metadata at 1731213582023 (+2 ms)Flushing 145f65b29b41c5cf34022ad77be79930/B: closing flushed file at 1731213582023Flushing 145f65b29b41c5cf34022ad77be79930/C: creating writer at 1731213582430 (+407 ms)Flushing 145f65b29b41c5cf34022ad77be79930/C: appending metadata at 1731213582432 (+2 ms)Flushing 145f65b29b41c5cf34022ad77be79930/C: closing flushed file at 1731213582432Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e280593: reopening flushed file at 1731213582838 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13672d79: reopening flushed file at 1731213582841 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3edfb7d: reopening flushed file at 1731213582844 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 145f65b29b41c5cf34022ad77be79930 in 1645ms, sequenceid=404, compaction requested=true at 1731213582847 (+3 ms)Writing region close event to WAL at 1731213582914 (+67 ms)Running coprocessor post-close hooks at 1731213582916 (+2 ms)Closed at 1731213582916 2024-11-10T04:39:42,918 INFO [RS_CLOSE_REGION-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:42,919 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=145f65b29b41c5cf34022ad77be79930, regionState=CLOSED 2024-11-10T04:39:42,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 because future has completed 2024-11-10T04:39:42,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-10T04:39:42,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 145f65b29b41c5cf34022ad77be79930, server=d1d9e3766cb1,45693,1731213391842 in 1.8700 sec 2024-11-10T04:39:42,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-10T04:39:42,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=145f65b29b41c5cf34022ad77be79930, UNASSIGN in 1.8730 sec 2024-11-10T04:39:42,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-10T04:39:42,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8760 sec 2024-11-10T04:39:42,925 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731213582925"}]},"ts":"1731213582925"} 2024-11-10T04:39:42,926 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T04:39:42,926 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T04:39:42,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8840 sec 2024-11-10T04:39:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-10T04:39:43,191 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:43,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T04:39:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:43,193 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-10T04:39:43,193 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:43,196 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,197 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C, FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/recovered.edits] 2024-11-10T04:39:43,199 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/1edf9f367fa142a4bc8b1fb78c4fa830 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/1edf9f367fa142a4bc8b1fb78c4fa830 2024-11-10T04:39:43,200 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8606f69849e94df49f48d065240399c8 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/8606f69849e94df49f48d065240399c8 2024-11-10T04:39:43,201 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ae5856d3d1a548dab47662bc456fc514 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/A/ae5856d3d1a548dab47662bc456fc514 2024-11-10T04:39:43,202 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/0f662d52b9ca4678bad2b0f956d1473c to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/0f662d52b9ca4678bad2b0f956d1473c 2024-11-10T04:39:43,203 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/7695a37b31a14d21a947e259db20e2c2 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/7695a37b31a14d21a947e259db20e2c2 2024-11-10T04:39:43,203 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8c51fe083d684dabb92d6e94b06d3a82 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/B/8c51fe083d684dabb92d6e94b06d3a82 2024-11-10T04:39:43,205 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/43bc43f4dc4f4c0bab24a38880bbfd0f to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/43bc43f4dc4f4c0bab24a38880bbfd0f 2024-11-10T04:39:43,206 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/88eb8862666f4767ba2f1fae1e12b346 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/88eb8862666f4767ba2f1fae1e12b346 2024-11-10T04:39:43,206 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/b550d58fb37843c0ac8ff3476e7cff84 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/C/b550d58fb37843c0ac8ff3476e7cff84 2024-11-10T04:39:43,208 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/recovered.edits/407.seqid to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930/recovered.edits/407.seqid 2024-11-10T04:39:43,209 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/default/TestAcidGuarantees/145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,209 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T04:39:43,209 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T04:39:43,209 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T04:39:43,211 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100880db167b744e579057e53c2eb016fd_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100880db167b744e579057e53c2eb016fd_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,212 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100a026a739e124d59b66b6cbed7a560e1_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100a026a739e124d59b66b6cbed7a560e1_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,213 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111017e304325d024ae8a6ff6508cc0acb1a_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111017e304325d024ae8a6ff6508cc0acb1a_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,214 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101a0531df228f456eb3c8f5796a87350f_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101a0531df228f456eb3c8f5796a87350f_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,215 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e8692f5089e40eda4b734ae1d8230fd_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101e8692f5089e40eda4b734ae1d8230fd_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,216 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111026cd40af7cc84d038b175ef39c9f0076_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111026cd40af7cc84d038b175ef39c9f0076_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,216 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111026f57b0b02e747d3824aab589f634a75_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111026f57b0b02e747d3824aab589f634a75_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,217 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103f70983655a84566ac1126e5afab2286_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103f70983655a84566ac1126e5afab2286_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,218 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046390c78d8a34ead9fec71fc7b2ce000_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111046390c78d8a34ead9fec71fc7b2ce000_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,219 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110685bae4d6f144260b93b1cb42f4083fe_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110685bae4d6f144260b93b1cb42f4083fe_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,220 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111081d82ac200f64cc3ab6774736be83d72_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111081d82ac200f64cc3ab6774736be83d72_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,220 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108c302c19da724bb791f25a13cbdecbfb_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108c302c19da724bb791f25a13cbdecbfb_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,221 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b8c9a73021844bb3b69e762536ae79b6_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b8c9a73021844bb3b69e762536ae79b6_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,222 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b8ceb6fb11604c54830e4bc4ac8b3be0_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b8ceb6fb11604c54830e4bc4ac8b3be0_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,223 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ba1a0aee982d49b28bfd32b8ebf27e31_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ba1a0aee982d49b28bfd32b8ebf27e31_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,224 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bc2808d4c7164c96b662862ae381f40c_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bc2808d4c7164c96b662862ae381f40c_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,224 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d541fcd4b8064c0f9c88f30ddb90632e_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d541fcd4b8064c0f9c88f30ddb90632e_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,225 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dee604829128490c870d36a83cebb505_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dee604829128490c870d36a83cebb505_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,226 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dfbceea96eff4490ac14843a8f04e4d6_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dfbceea96eff4490ac14843a8f04e4d6_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,227 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e9265be055894b499f7d394db40e1111_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e9265be055894b499f7d394db40e1111_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,228 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f120915f86ff476fa7b81d39bf423cf0_145f65b29b41c5cf34022ad77be79930 to hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f120915f86ff476fa7b81d39bf423cf0_145f65b29b41c5cf34022ad77be79930 2024-11-10T04:39:43,228 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T04:39:43,230 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:43,232 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T04:39:43,234 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T04:39:43,234 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:43,234 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T04:39:43,235 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731213583234"}]},"ts":"9223372036854775807"} 2024-11-10T04:39:43,236 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T04:39:43,236 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 145f65b29b41c5cf34022ad77be79930, NAME => 'TestAcidGuarantees,,1731213548130.145f65b29b41c5cf34022ad77be79930.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T04:39:43,236 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T04:39:43,236 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731213583236"}]},"ts":"9223372036854775807"} 2024-11-10T04:39:43,237 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T04:39:43,238 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T04:39:43,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-10T04:39:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-10T04:39:43,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T04:39:43,302 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T04:39:43,310 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 240), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 356), ProcessCount=11 (was 11), AvailableMemoryMB=6728 (was 6857) 2024-11-10T04:39:43,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T04:39:43,310 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:39:43,311 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:39:43,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:43,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:43,311 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:39:43,311 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T04:39:43,311 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=370755489, stopped=false 2024-11-10T04:39:43,311 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d1d9e3766cb1,42929,1731213391190 2024-11-10T04:39:43,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:39:43,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:39:43,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:39:43,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:39:43,313 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:39:43,314 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:39:43,314 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:39:43,314 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:39:43,314 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:39:43,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:43,315 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd1d9e3766cb1,45693,1731213391842' ***** 2024-11-10T04:39:43,315 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T04:39:43,315 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:39:43,315 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:39:43,315 INFO [RS:0;d1d9e3766cb1:45693 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:39:43,316 INFO [RS:0;d1d9e3766cb1:45693 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:39:43,316 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(959): stopping server d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:43,316 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:39:43,316 INFO [RS:0;d1d9e3766cb1:45693 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d1d9e3766cb1:45693. 2024-11-10T04:39:43,316 DEBUG [RS:0;d1d9e3766cb1:45693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:39:43,316 DEBUG [RS:0;d1d9e3766cb1:45693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:43,316 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:39:43,316 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:39:43,316 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:39:43,316 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T04:39:43,317 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T04:39:43,317 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T04:39:43,317 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:39:43,317 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:39:43,317 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:39:43,317 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:39:43,317 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:39:43,317 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:39:43,317 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T04:39:43,338 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/info/1dd4af1f3cf640ed845f580f284b73b8 is 95, key is TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc./info:/1731213430350/DeleteFamily/seqid=0 2024-11-10T04:39:43,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742442_1618 (size=6514) 2024-11-10T04:39:43,377 INFO [regionserver/d1d9e3766cb1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:39:43,517 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:39:43,709 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T04:39:43,709 INFO [regionserver/d1d9e3766cb1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T04:39:43,718 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:39:43,741 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/info/1dd4af1f3cf640ed845f580f284b73b8 2024-11-10T04:39:43,760 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/ns/b9cec5e2a33347c1bec14fa8c43c554a is 93, key is TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc./ns:/1731213430311/DeleteFamily/seqid=0 2024-11-10T04:39:43,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742443_1619 (size=5906) 2024-11-10T04:39:43,918 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:39:44,118 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:39:44,163 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/ns/b9cec5e2a33347c1bec14fa8c43c554a 2024-11-10T04:39:44,181 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/rep_barrier/567349edc8bf484088ace8b66447aed2 is 102, key is TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc./rep_barrier:/1731213430311/DeleteFamily/seqid=0 2024-11-10T04:39:44,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742444_1620 (size=6025) 2024-11-10T04:39:44,318 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:39:44,518 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:39:44,584 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/rep_barrier/567349edc8bf484088ace8b66447aed2 2024-11-10T04:39:44,603 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/table/f961e085d0ae423d9b00ed4ebdc8e6e1 is 96, key is TestAcidGuarantees,,1731213395260.6d2efd97fd4ccb14e877b384767c0ffc./table:/1731213430311/DeleteFamily/seqid=0 2024-11-10T04:39:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742445_1621 (size=5936) 2024-11-10T04:39:44,719 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:39:44,919 DEBUG [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T04:39:45,007 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/table/f961e085d0ae423d9b00ed4ebdc8e6e1 2024-11-10T04:39:45,010 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/info/1dd4af1f3cf640ed845f580f284b73b8 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/info/1dd4af1f3cf640ed845f580f284b73b8 2024-11-10T04:39:45,013 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/info/1dd4af1f3cf640ed845f580f284b73b8, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T04:39:45,013 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/ns/b9cec5e2a33347c1bec14fa8c43c554a as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/ns/b9cec5e2a33347c1bec14fa8c43c554a 2024-11-10T04:39:45,016 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/ns/b9cec5e2a33347c1bec14fa8c43c554a, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T04:39:45,016 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/rep_barrier/567349edc8bf484088ace8b66447aed2 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/rep_barrier/567349edc8bf484088ace8b66447aed2 2024-11-10T04:39:45,019 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/rep_barrier/567349edc8bf484088ace8b66447aed2, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T04:39:45,019 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/.tmp/table/f961e085d0ae423d9b00ed4ebdc8e6e1 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/table/f961e085d0ae423d9b00ed4ebdc8e6e1 2024-11-10T04:39:45,022 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/table/f961e085d0ae423d9b00ed4ebdc8e6e1, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T04:39:45,023 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false 2024-11-10T04:39:45,026 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T04:39:45,027 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:39:45,027 INFO [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:39:45,027 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731213583317Running coprocessor pre-close hooks at 1731213583317Disabling compacts and flushes for region at 1731213583317Disabling writes for close at 1731213583317Obtaining lock to block concurrent updates at 1731213583317Preparing flush snapshotting stores in 1588230740 at 1731213583317Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731213583318 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731213583318Flushing 1588230740/info: creating writer at 1731213583318Flushing 1588230740/info: appending metadata at 1731213583337 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731213583337Flushing 1588230740/ns: creating writer at 1731213583745 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731213583760 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731213583760Flushing 1588230740/rep_barrier: creating writer at 1731213584166 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731213584180 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731213584180Flushing 1588230740/table: creating writer at 1731213584587 (+407 ms)Flushing 1588230740/table: appending metadata at 1731213584603 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731213584603Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51c5b6e9: reopening flushed file at 1731213585010 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@207d71ca: reopening flushed file at 1731213585013 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13437068: reopening flushed file at 1731213585016 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1179765f: reopening flushed file at 1731213585019 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false at 1731213585023 (+4 ms)Writing region close event to WAL at 1731213585024 (+1 ms)Running coprocessor post-close hooks at 1731213585027 (+3 ms)Closed at 1731213585027 2024-11-10T04:39:45,027 DEBUG [RS_CLOSE_META-regionserver/d1d9e3766cb1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T04:39:45,119 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(976): stopping server d1d9e3766cb1,45693,1731213391842; all regions closed. 2024-11-10T04:39:45,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741834_1010 (size=25344) 2024-11-10T04:39:45,125 DEBUG [RS:0;d1d9e3766cb1:45693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/oldWALs 2024-11-10T04:39:45,126 INFO [RS:0;d1d9e3766cb1:45693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d1d9e3766cb1%2C45693%2C1731213391842.meta:.meta(num 1731213394857) 2024-11-10T04:39:45,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741832_1008 (size=14017212) 2024-11-10T04:39:45,129 DEBUG [RS:0;d1d9e3766cb1:45693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/oldWALs 2024-11-10T04:39:45,129 INFO [RS:0;d1d9e3766cb1:45693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d1d9e3766cb1%2C45693%2C1731213391842:(num 1731213393861) 2024-11-10T04:39:45,129 DEBUG [RS:0;d1d9e3766cb1:45693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:39:45,129 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:39:45,129 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:39:45,129 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.ChoreService(370): Chore service for: regionserver/d1d9e3766cb1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T04:39:45,129 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:39:45,129 INFO [regionserver/d1d9e3766cb1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:39:45,130 INFO [RS:0;d1d9e3766cb1:45693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45693 2024-11-10T04:39:45,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1d9e3766cb1,45693,1731213391842 2024-11-10T04:39:45,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:39:45,134 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:39:45,136 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1d9e3766cb1,45693,1731213391842] 2024-11-10T04:39:45,137 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d1d9e3766cb1,45693,1731213391842 already deleted, retry=false 2024-11-10T04:39:45,138 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d1d9e3766cb1,45693,1731213391842 expired; onlineServers=0 2024-11-10T04:39:45,138 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd1d9e3766cb1,42929,1731213391190' ***** 2024-11-10T04:39:45,138 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T04:39:45,138 INFO [M:0;d1d9e3766cb1:42929 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:39:45,138 INFO [M:0;d1d9e3766cb1:42929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:39:45,138 DEBUG [M:0;d1d9e3766cb1:42929 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T04:39:45,138 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T04:39:45,138 DEBUG [M:0;d1d9e3766cb1:42929 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T04:39:45,138 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731213393610 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.small.0-1731213393610,5,FailOnTimeoutGroup] 2024-11-10T04:39:45,138 DEBUG [master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731213393609 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1d9e3766cb1:0:becomeActiveMaster-HFileCleaner.large.0-1731213393609,5,FailOnTimeoutGroup] 2024-11-10T04:39:45,139 INFO [M:0;d1d9e3766cb1:42929 {}] hbase.ChoreService(370): Chore service for: master/d1d9e3766cb1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T04:39:45,139 INFO [M:0;d1d9e3766cb1:42929 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:39:45,139 DEBUG [M:0;d1d9e3766cb1:42929 {}] master.HMaster(1795): Stopping service threads 2024-11-10T04:39:45,139 INFO [M:0;d1d9e3766cb1:42929 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T04:39:45,139 INFO [M:0;d1d9e3766cb1:42929 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:39:45,139 ERROR [M:0;d1d9e3766cb1:42929 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:38385 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:38385,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T04:39:45,139 INFO [M:0;d1d9e3766cb1:42929 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T04:39:45,140 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T04:39:45,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T04:39:45,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:39:45,140 DEBUG [M:0;d1d9e3766cb1:42929 {}] zookeeper.ZKUtil(347): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T04:39:45,140 WARN [M:0;d1d9e3766cb1:42929 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T04:39:45,141 INFO [M:0;d1d9e3766cb1:42929 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/.lastflushedseqids 2024-11-10T04:39:45,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742446_1622 (size=72) 2024-11-10T04:39:45,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:39:45,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1019077678c0001, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:39:45,237 INFO [RS:0;d1d9e3766cb1:45693 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:39:45,237 INFO [RS:0;d1d9e3766cb1:45693 {}] regionserver.HRegionServer(1031): Exiting; stopping=d1d9e3766cb1,45693,1731213391842; zookeeper connection closed. 2024-11-10T04:39:45,238 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@49a62903 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@49a62903 2024-11-10T04:39:45,238 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T04:39:45,551 INFO [M:0;d1d9e3766cb1:42929 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T04:39:45,551 INFO [M:0;d1d9e3766cb1:42929 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T04:39:45,551 DEBUG [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:39:45,551 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:39:45,551 DEBUG [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:39:45,551 DEBUG [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:39:45,551 DEBUG [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:39:45,551 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=732.85 KB heapSize=898.76 KB 2024-11-10T04:39:45,567 DEBUG [M:0;d1d9e3766cb1:42929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/154c716933e342db87270338e821ebe5 is 82, key is hbase:meta,,1/info:regioninfo/1731213394948/Put/seqid=0 2024-11-10T04:39:45,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742447_1623 (size=5672) 2024-11-10T04:39:45,971 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2036 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/154c716933e342db87270338e821ebe5 2024-11-10T04:39:45,994 DEBUG [M:0;d1d9e3766cb1:42929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/614bd8352fe147d99f99a494e1385a19 is 2276, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1731213433541/Put/seqid=0 2024-11-10T04:39:45,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742448_1624 (size=44896) 2024-11-10T04:39:46,398 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732.29 KB at sequenceid=2036 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/614bd8352fe147d99f99a494e1385a19 2024-11-10T04:39:46,402 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 614bd8352fe147d99f99a494e1385a19 2024-11-10T04:39:46,419 DEBUG [M:0;d1d9e3766cb1:42929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4f05e68468c64047af3925a4d36e0c04 is 69, key is d1d9e3766cb1,45693,1731213391842/rs:state/1731213393628/Put/seqid=0 2024-11-10T04:39:46,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073742449_1625 (size=5156) 2024-11-10T04:39:46,823 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2036 (bloomFilter=true), to=hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4f05e68468c64047af3925a4d36e0c04 2024-11-10T04:39:46,826 DEBUG [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/154c716933e342db87270338e821ebe5 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/154c716933e342db87270338e821ebe5 2024-11-10T04:39:46,829 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/154c716933e342db87270338e821ebe5, entries=8, sequenceid=2036, filesize=5.5 K 2024-11-10T04:39:46,830 DEBUG [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/614bd8352fe147d99f99a494e1385a19 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/614bd8352fe147d99f99a494e1385a19 2024-11-10T04:39:46,832 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 614bd8352fe147d99f99a494e1385a19 2024-11-10T04:39:46,832 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/614bd8352fe147d99f99a494e1385a19, entries=174, sequenceid=2036, filesize=43.8 K 2024-11-10T04:39:46,833 DEBUG [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4f05e68468c64047af3925a4d36e0c04 as hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4f05e68468c64047af3925a4d36e0c04 2024-11-10T04:39:46,836 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38385/user/jenkins/test-data/7831c23a-8d5e-213e-285c-d263e6884ea8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4f05e68468c64047af3925a4d36e0c04, entries=1, sequenceid=2036, filesize=5.0 K 2024-11-10T04:39:46,836 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegion(3140): Finished flush of dataSize ~732.85 KB/750437, heapSize ~898.46 KB/920024, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2036, compaction requested=false 2024-11-10T04:39:46,838 INFO [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:39:46,838 DEBUG [M:0;d1d9e3766cb1:42929 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731213585551Disabling compacts and flushes for region at 1731213585551Disabling writes for close at 1731213585551Obtaining lock to block concurrent updates at 1731213585551Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731213585551Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=750437, getHeapSize=920264, getOffHeapSize=0, getCellsCount=2235 at 1731213585551Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731213585552 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731213585552Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731213585567 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731213585567Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731213585975 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731213585993 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731213585993Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731213586402 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731213586419 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731213586419Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d82bc66: reopening flushed file at 1731213586826 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f0abe17: reopening flushed file at 1731213586829 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@597b4535: reopening flushed file at 1731213586832 (+3 ms)Finished flush of dataSize ~732.85 KB/750437, heapSize ~898.46 KB/920024, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1285ms, sequenceid=2036, compaction requested=false at 1731213586836 (+4 ms)Writing region close event to WAL at 1731213586838 (+2 ms)Closed at 1731213586838 2024-11-10T04:39:46,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45227 is added to blk_1073741830_1006 (size=883992) 2024-11-10T04:39:46,841 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:39:46,841 INFO [M:0;d1d9e3766cb1:42929 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T04:39:46,841 INFO [M:0;d1d9e3766cb1:42929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42929 2024-11-10T04:39:46,841 INFO [M:0;d1d9e3766cb1:42929 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:39:46,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:39:46,943 INFO [M:0;d1d9e3766cb1:42929 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:39:46,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1019077678c0000, quorum=127.0.0.1:57257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:39:46,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T04:39:46,950 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:39:46,950 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:39:46,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:39:46,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/hadoop.log.dir/,STOPPED} 2024-11-10T04:39:46,953 WARN [BP-1091589648-172.17.0.2-1731213388437 heartbeating to localhost/127.0.0.1:38385 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T04:39:46,953 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T04:39:46,953 WARN [BP-1091589648-172.17.0.2-1731213388437 heartbeating to localhost/127.0.0.1:38385 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1091589648-172.17.0.2-1731213388437 (Datanode Uuid c7686c1b-b994-4981-a73c-a823daec5640) service to localhost/127.0.0.1:38385 2024-11-10T04:39:46,953 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T04:39:46,955 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff/data/data1/current/BP-1091589648-172.17.0.2-1731213388437 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:39:46,956 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/cluster_0faf3bcc-01cf-fb0e-08f9-c9c539de60ff/data/data2/current/BP-1091589648-172.17.0.2-1731213388437 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T04:39:46,956 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T04:39:46,964 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T04:39:46,965 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T04:39:46,965 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T04:39:46,965 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T04:39:46,965 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5dc84d12-6ed1-ddfd-9f39-a6e808d60856/hadoop.log.dir/,STOPPED} 2024-11-10T04:39:46,980 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T04:39:47,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down